Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / trl /pr_5607 /en /_app /immutable /nodes /60.33dcbf6b.js
HuggingFaceDocBuilder's picture
HuggingFaceDocBuilder
download
raw
60.8 kB
import{s as ut,o as ht,n as V}from"../chunks/scheduler.7b731bd4.js";import{S as ft,i as jt,e as j,s as o,c as p,h as It,a as I,d as a,b as i,f as Ut,g as w,j as $,k as N,l as $t,m as M,n as d,t as T,o as y,p as J}from"../chunks/index.cc268345.js";import{C as gt,H as v,E as bt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f0d99f98.js";import{C as g}from"../chunks/CodeBlock.169a125f.js";import{H as Xl,a as G}from"../chunks/HfOption.9f04abd1.js";function Ct(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTJDJTIwR1JQT0NvbmZpZyUwQWZyb20lMjB0cmwucmV3YXJkcyUyMGltcG9ydCUyMGFjY3VyYWN5X3Jld2FyZCUwQSUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIydHJsLWxpYiUyRkRlZXBNYXRoLTEwM0slMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBHUlBPVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRCUyMlF3ZW4lMkZRd2VuMi41LTdCJTIyJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzREdSUE9Db25maWcodXNlX3ZsbG0lM0RUcnVlJTJDJTIwdmxsbV9tb2RlJTNEJTIyc2VydmVyJTIyKSUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRGFjY3VyYWN5X3Jld2FyZCUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0RkYXRhc2V0JTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer, GRPOConfig
<span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> accuracy_reward
dataset = load_dataset(<span class="hljs-string">&quot;trl-lib/DeepMath-103K&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
trainer = GRPOTrainer(
 model=<span class="hljs-string">&quot;Qwen/Qwen2.5-7B&quot;</span>,
 args=GRPOConfig(use_vllm=<span class="hljs-literal">True</span>, vllm_mode=<span class="hljs-string">&quot;server&quot;</span>),
 reward_funcs=accuracy_reward,
 train_dataset=dataset,
)
trainer.train()`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Zt(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHRybC5leHBlcmltZW50YWwub25saW5lX2RwbyUyMGltcG9ydCUyME9ubGluZURQT0NvbmZpZyUyQyUyME9ubGluZURQT1RyYWluZXIlMEFmcm9tJTIwdHJsLnJld2FyZHMlMjBpbXBvcnQlMjBhY2N1cmFjeV9yZXdhcmQlMEElMEFkYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMnRybC1saWIlMkZEZWVwTWF0aC0xMDNLJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEElMEF0cmFpbmVyJTIwJTNEJTIwT25saW5lRFBPVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRCUyMlF3ZW4lMkZRd2VuMi41LTdCJTIyJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRE9ubGluZURQT0NvbmZpZyh1c2VfdmxsbSUzRFRydWUlMkMlMjB2bGxtX21vZGUlM0QlMjJzZXJ2ZXIlMjIpJTJDJTBBJTIwJTIwJTIwJTIwcmV3YXJkX2Z1bmNzJTNEYWNjdXJhY3lfcmV3YXJkJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> trl.experimental.online_dpo <span class="hljs-keyword">import</span> OnlineDPOConfig, OnlineDPOTrainer
<span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> accuracy_reward
dataset = load_dataset(<span class="hljs-string">&quot;trl-lib/DeepMath-103K&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
trainer = OnlineDPOTrainer(
 model=<span class="hljs-string">&quot;Qwen/Qwen2.5-7B&quot;</span>,
 args=OnlineDPOConfig(use_vllm=<span class="hljs-literal">True</span>, vllm_mode=<span class="hljs-string">&quot;server&quot;</span>),
 reward_funcs=accuracy_reward,
 train_dataset=dataset,
)
trainer.train()`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function At(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHRybC5leHBlcmltZW50YWwubmFzaF9tZCUyMGltcG9ydCUyME5hc2hNRENvbmZpZyUyQyUyME5hc2hNRFRyYWluZXIlMEFmcm9tJTIwdHJsLnJld2FyZHMlMjBpbXBvcnQlMjBhY2N1cmFjeV9yZXdhcmQlMEElMEFkYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMnRybC1saWIlMkZEZWVwTWF0aC0xMDNLJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEElMEF0cmFpbmVyJTIwJTNEJTIwTmFzaE1EVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRCUyMlF3ZW4lMkZRd2VuMi41LTdCJTIyJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRE5hc2hNRENvbmZpZyh1c2VfdmxsbSUzRFRydWUlMkMlMjB2bGxtX21vZGUlM0QlMjJzZXJ2ZXIlMjIpJTJDJTBBJTIwJTIwJTIwJTIwcmV3YXJkX2Z1bmNzJTNEYWNjdXJhY3lfcmV3YXJkJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> trl.experimental.nash_md <span class="hljs-keyword">import</span> NashMDConfig, NashMDTrainer
<span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> accuracy_reward
dataset = load_dataset(<span class="hljs-string">&quot;trl-lib/DeepMath-103K&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
trainer = NashMDTrainer(
 model=<span class="hljs-string">&quot;Qwen/Qwen2.5-7B&quot;</span>,
 args=NashMDConfig(use_vllm=<span class="hljs-literal">True</span>, vllm_mode=<span class="hljs-string">&quot;server&quot;</span>),
 reward_funcs=accuracy_reward,
 train_dataset=dataset,
)
trainer.train()`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function vt(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHRybC5leHBlcmltZW50YWwueHBvJTIwaW1wb3J0JTIwWFBPVHJhaW5lciUyQyUyMFhQT0NvbmZpZyUwQWZyb20lMjB0cmwucmV3YXJkcyUyMGltcG9ydCUyMGFjY3VyYWN5X3Jld2FyZCUwQSUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIydHJsLWxpYiUyRkRlZXBNYXRoLTEwM0slMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBYUE9UcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEJTIyUXdlbiUyRlF3ZW4yLjUtN0IlMjIlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEWFBPQ29uZmlnKHVzZV92bGxtJTNEVHJ1ZSUyQyUyMHZsbG1fbW9kZSUzRCUyMnNlcnZlciUyMiklMkMlMEElMjAlMjAlMjAlMjByZXdhcmRfZnVuY3MlM0RhY2N1cmFjeV9yZXdhcmQlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCUyQyUwQSklMEElMEF0cmFpbmVyLnRyYWluKCk=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> trl.experimental.xpo <span class="hljs-keyword">import</span> XPOTrainer, XPOConfig
<span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> accuracy_reward
dataset = load_dataset(<span class="hljs-string">&quot;trl-lib/DeepMath-103K&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
trainer = XPOTrainer(
 model=<span class="hljs-string">&quot;Qwen/Qwen2.5-7B&quot;</span>,
 args=XPOConfig(use_vllm=<span class="hljs-literal">True</span>, vllm_mode=<span class="hljs-string">&quot;server&quot;</span>),
 reward_funcs=accuracy_reward,
 train_dataset=dataset,
)
trainer.train()`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Gt(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHRybCUyMGltcG9ydCUyMFJMT09UcmFpbmVyJTJDJTIwUkxPT0NvbmZpZyUwQWZyb20lMjB0cmwucmV3YXJkcyUyMGltcG9ydCUyMGFjY3VyYWN5X3Jld2FyZCUwQSUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIydHJsLWxpYiUyRkRlZXBNYXRoLTEwM0slMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBSTE9PVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRCUyMlF3ZW4lMkZRd2VuMi41LTdCJTIyJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRFJMT09Db25maWcodXNlX3ZsbG0lM0RUcnVlJTJDJTIwdmxsbV9tb2RlJTNEJTIyc2VydmVyJTIyKSUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRGFjY3VyYWN5X3Jld2FyZCUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0RkYXRhc2V0JTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> RLOOTrainer, RLOOConfig
<span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> accuracy_reward
dataset = load_dataset(<span class="hljs-string">&quot;trl-lib/DeepMath-103K&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
trainer = RLOOTrainer(
 model=<span class="hljs-string">&quot;Qwen/Qwen2.5-7B&quot;</span>,
 args=RLOOConfig(use_vllm=<span class="hljs-literal">True</span>, vllm_mode=<span class="hljs-string">&quot;server&quot;</span>),
 reward_funcs=accuracy_reward,
 train_dataset=dataset,
)
trainer.train()`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Vt(U){let t,r,l,c,u,b,h,C,f,Z;return t=new G({props:{id:"vllm examples",option:"GRPO",$$slots:{default:[Ct]},$$scope:{ctx:U}}}),l=new G({props:{id:"vllm examples",option:"OnlineDPO",$$slots:{default:[Zt]},$$scope:{ctx:U}}}),u=new G({props:{id:"vllm examples",option:"NashMD",$$slots:{default:[At]},$$scope:{ctx:U}}}),h=new G({props:{id:"vllm examples",option:"XPO",$$slots:{default:[vt]},$$scope:{ctx:U}}}),f=new G({props:{id:"vllm examples",option:"RLOO",$$slots:{default:[Gt]},$$scope:{ctx:U}}}),{c(){p(t.$$.fragment),r=o(),p(l.$$.fragment),c=o(),p(u.$$.fragment),b=o(),p(h.$$.fragment),C=o(),p(f.$$.fragment)},l(s){w(t.$$.fragment,s),r=i(s),w(l.$$.fragment,s),c=i(s),w(u.$$.fragment,s),b=i(s),w(h.$$.fragment,s),C=i(s),w(f.$$.fragment,s)},m(s,m){d(t,s,m),M(s,r,m),d(l,s,m),M(s,c,m),d(u,s,m),M(s,b,m),d(h,s,m),M(s,C,m),d(f,s,m),Z=!0},p(s,m){const B={};m&2&&(B.$$scope={dirty:m,ctx:s}),t.$set(B);const R={};m&2&&(R.$$scope={dirty:m,ctx:s}),l.$set(R);const A={};m&2&&(A.$$scope={dirty:m,ctx:s}),u.$set(A);const _={};m&2&&(_.$$scope={dirty:m,ctx:s}),h.$set(_);const E={};m&2&&(E.$$scope={dirty:m,ctx:s}),f.$set(E)},i(s){Z||(T(t.$$.fragment,s),T(l.$$.fragment,s),T(u.$$.fragment,s),T(h.$$.fragment,s),T(f.$$.fragment,s),Z=!0)},o(s){y(t.$$.fragment,s),y(l.$$.fragment,s),y(u.$$.fragment,s),y(h.$$.fragment,s),y(f.$$.fragment,s),Z=!1},d(s){s&&(a(r),a(c),a(b),a(C)),J(t,s),J(l,s),J(u,s),J(h,s),J(f,s)}}}function Rt(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwR1JQT0NvbmZpZyglMEElMjAlMjAlMjAlMjAuLi4lMkMlMEElMjAlMjAlMjAlMjB1c2VfdmxsbSUzRFRydWUlMkMlMjAlMjAlMjMlMjB2bGxtX21vZGUlM0QlMjJjb2xvY2F0ZSUyMiUyMGJ5JTIwZGVmYXVsdCUwQSk=",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig
training_args = GRPOConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>, <span class="hljs-comment"># vllm_mode=&quot;colocate&quot; by default</span>
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Et(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwub25saW5lX2RwbyUyMGltcG9ydCUyME9ubGluZURQT0NvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBPbmxpbmVEUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwLi4uJTJDJTBBJTIwJTIwJTIwJTIwdXNlX3ZsbG0lM0RUcnVlJTJDJTIwJTIwJTIzJTIwdmxsbV9tb2RlJTNEJTIyY29sb2NhdGUlMjIlMjBieSUyMGRlZmF1bHQlMEEp",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.online_dpo <span class="hljs-keyword">import</span> OnlineDPOConfig
training_args = OnlineDPOConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>, <span class="hljs-comment"># vllm_mode=&quot;colocate&quot; by default</span>
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Bt(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwubmFzaF9tZCUyMGltcG9ydCUyME5hc2hNRENvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBOYXNoTURDb25maWcoJTBBJTIwJTIwJTIwJTIwLi4uJTJDJTBBJTIwJTIwJTIwJTIwdXNlX3ZsbG0lM0RUcnVlJTJDJTIwJTIwJTIzJTIwdmxsbV9tb2RlJTNEJTIyY29sb2NhdGUlMjIlMjBieSUyMGRlZmF1bHQlMEEp",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.nash_md <span class="hljs-keyword">import</span> NashMDConfig
training_args = NashMDConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>, <span class="hljs-comment"># vllm_mode=&quot;colocate&quot; by default</span>
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function _t(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwueHBvJTIwaW1wb3J0JTIwWFBPQ29uZmlnJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMFhQT0NvbmZpZyglMEElMjAlMjAlMjAlMjAuLi4lMkMlMEElMjAlMjAlMjAlMjB1c2VfdmxsbSUzRFRydWUlMkMlMjAlMjAlMjMlMjB2bGxtX21vZGUlM0QlMjJjb2xvY2F0ZSUyMiUyMGJ5JTIwZGVmYXVsdCUwQSk=",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.xpo <span class="hljs-keyword">import</span> XPOConfig
training_args = XPOConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>, <span class="hljs-comment"># vllm_mode=&quot;colocate&quot; by default</span>
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Lt(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFJMT09Db25maWclMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwUkxPT0NvbmZpZyglMEElMjAlMjAlMjAlMjAuLi4lMkMlMEElMjAlMjAlMjAlMjB1c2VfdmxsbSUzRFRydWUlMkMlMjAlMjAlMjMlMjB2bGxtX21vZGUlM0QlMjJjb2xvY2F0ZSUyMiUyMGJ5JTIwZGVmYXVsdCUwQSk=",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> RLOOConfig
training_args = RLOOConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>, <span class="hljs-comment"># vllm_mode=&quot;colocate&quot; by default</span>
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function kt(U){let t,r,l,c,u,b,h,C,f,Z;return t=new G({props:{id:"vllm examples",option:"GRPO",$$slots:{default:[Rt]},$$scope:{ctx:U}}}),l=new G({props:{id:"vllm examples",option:"OnlineDPO",$$slots:{default:[Et]},$$scope:{ctx:U}}}),u=new G({props:{id:"vllm examples",option:"NashMD",$$slots:{default:[Bt]},$$scope:{ctx:U}}}),h=new G({props:{id:"vllm examples",option:"XPO",$$slots:{default:[_t]},$$scope:{ctx:U}}}),f=new G({props:{id:"vllm examples",option:"RLOO",$$slots:{default:[Lt]},$$scope:{ctx:U}}}),{c(){p(t.$$.fragment),r=o(),p(l.$$.fragment),c=o(),p(u.$$.fragment),b=o(),p(h.$$.fragment),C=o(),p(f.$$.fragment)},l(s){w(t.$$.fragment,s),r=i(s),w(l.$$.fragment,s),c=i(s),w(u.$$.fragment,s),b=i(s),w(h.$$.fragment,s),C=i(s),w(f.$$.fragment,s)},m(s,m){d(t,s,m),M(s,r,m),d(l,s,m),M(s,c,m),d(u,s,m),M(s,b,m),d(h,s,m),M(s,C,m),d(f,s,m),Z=!0},p(s,m){const B={};m&2&&(B.$$scope={dirty:m,ctx:s}),t.$set(B);const R={};m&2&&(R.$$scope={dirty:m,ctx:s}),l.$set(R);const A={};m&2&&(A.$$scope={dirty:m,ctx:s}),u.$set(A);const _={};m&2&&(_.$$scope={dirty:m,ctx:s}),h.$set(_);const E={};m&2&&(E.$$scope={dirty:m,ctx:s}),f.$set(E)},i(s){Z||(T(t.$$.fragment,s),T(l.$$.fragment,s),T(u.$$.fragment,s),T(h.$$.fragment,s),T(f.$$.fragment,s),Z=!0)},o(s){y(t.$$.fragment,s),y(l.$$.fragment,s),y(u.$$.fragment,s),y(h.$$.fragment,s),y(f.$$.fragment,s),Z=!1},d(s){s&&(a(r),a(c),a(b),a(C)),J(t,s),J(l,s),J(u,s),J(h,s),J(f,s)}}}function Wt(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwR1JQT0NvbmZpZyglMEElMjAlMjAlMjAlMjAuLi4lMkMlMEElMjAlMjAlMjAlMjB1c2VfdmxsbSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjB2bGxtX21vZGUlM0QlMjJzZXJ2ZXIlMjIlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig
training_args = GRPOConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>,
 vllm_mode=<span class="hljs-string">&quot;server&quot;</span>,
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Xt(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwub25saW5lX2RwbyUyMGltcG9ydCUyME9ubGluZURQT0NvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBPbmxpbmVEUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwLi4uJTJDJTBBJTIwJTIwJTIwJTIwdXNlX3ZsbG0lM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwdmxsbV9tb2RlJTNEJTIyc2VydmVyJTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.online_dpo <span class="hljs-keyword">import</span> OnlineDPOConfig
training_args = OnlineDPOConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>,
 vllm_mode=<span class="hljs-string">&quot;server&quot;</span>,
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function St(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwubmFzaF9tZCUyMGltcG9ydCUyME5hc2hNRENvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBOYXNoTURDb25maWcoJTBBJTIwJTIwJTIwJTIwLi4uJTJDJTBBJTIwJTIwJTIwJTIwdXNlX3ZsbG0lM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwdmxsbV9tb2RlJTNEJTIyc2VydmVyJTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.nash_md <span class="hljs-keyword">import</span> NashMDConfig
training_args = NashMDConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>,
 vllm_mode=<span class="hljs-string">&quot;server&quot;</span>,
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Ft(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwueHBvJTIwaW1wb3J0JTIwWFBPQ29uZmlnJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMFhQT0NvbmZpZyglMEElMjAlMjAlMjAlMjAuLi4lMkMlMEElMjAlMjAlMjAlMjB1c2VfdmxsbSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjB2bGxtX21vZGUlM0QlMjJzZXJ2ZXIlMjIlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.xpo <span class="hljs-keyword">import</span> XPOConfig
training_args = XPOConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>,
 vllm_mode=<span class="hljs-string">&quot;server&quot;</span>,
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Qt(U){let t,r;return t=new g({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFJMT09Db25maWclMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwUkxPT0NvbmZpZyglMEElMjAlMjAlMjAlMjAuLi4lMkMlMEElMjAlMjAlMjAlMjB1c2VfdmxsbSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjB2bGxtX21vZGUlM0QlMjJzZXJ2ZXIlMjIlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> RLOOConfig
training_args = RLOOConfig(
 ...,
 use_vllm=<span class="hljs-literal">True</span>,
 vllm_mode=<span class="hljs-string">&quot;server&quot;</span>,
)`,wrap:!1}}),{c(){p(t.$$.fragment)},l(l){w(t.$$.fragment,l)},m(l,c){d(t,l,c),r=!0},p:V,i(l){r||(T(t.$$.fragment,l),r=!0)},o(l){y(t.$$.fragment,l),r=!1},d(l){J(t,l)}}}function Nt(U){let t,r,l,c,u,b,h,C,f,Z;return t=new G({props:{id:"vllm examples",option:"GRPO",$$slots:{default:[Wt]},$$scope:{ctx:U}}}),l=new G({props:{id:"vllm examples",option:"OnlineDPO",$$slots:{default:[Xt]},$$scope:{ctx:U}}}),u=new G({props:{id:"vllm examples",option:"NashMD",$$slots:{default:[St]},$$scope:{ctx:U}}}),h=new G({props:{id:"vllm examples",option:"XPO",$$slots:{default:[Ft]},$$scope:{ctx:U}}}),f=new G({props:{id:"vllm examples",option:"RLOO",$$slots:{default:[Qt]},$$scope:{ctx:U}}}),{c(){p(t.$$.fragment),r=o(),p(l.$$.fragment),c=o(),p(u.$$.fragment),b=o(),p(h.$$.fragment),C=o(),p(f.$$.fragment)},l(s){w(t.$$.fragment,s),r=i(s),w(l.$$.fragment,s),c=i(s),w(u.$$.fragment,s),b=i(s),w(h.$$.fragment,s),C=i(s),w(f.$$.fragment,s)},m(s,m){d(t,s,m),M(s,r,m),d(l,s,m),M(s,c,m),d(u,s,m),M(s,b,m),d(h,s,m),M(s,C,m),d(f,s,m),Z=!0},p(s,m){const B={};m&2&&(B.$$scope={dirty:m,ctx:s}),t.$set(B);const R={};m&2&&(R.$$scope={dirty:m,ctx:s}),l.$set(R);const A={};m&2&&(A.$$scope={dirty:m,ctx:s}),u.$set(A);const _={};m&2&&(_.$$scope={dirty:m,ctx:s}),h.$set(_);const E={};m&2&&(E.$$scope={dirty:m,ctx:s}),f.$set(E)},i(s){Z||(T(t.$$.fragment,s),T(l.$$.fragment,s),T(u.$$.fragment,s),T(h.$$.fragment,s),T(f.$$.fragment,s),Z=!0)},o(s){y(t.$$.fragment,s),y(l.$$.fragment,s),y(u.$$.fragment,s),y(h.$$.fragment,s),y(f.$$.fragment,s),Z=!1},d(s){s&&(a(r),a(c),a(b),a(C)),J(t,s),J(l,s),J(u,s),J(h,s),J(f,s)}}}function xt(U){let t,r,l,c,u,b,h,C,f,Z="This document will guide you through the process of using vLLM with TRL for faster generation in online methods like GRPO and Online DPO. We first summarize a tl;dr on how to use vLLM with TRL, and then we will go into the details of how it works under the hood.",s,m,B="<p>TRL currently only supports vLLM versions from <code>0.11.0</code> to <code>0.18.0</code>. Please ensure you have a version in this range installed to avoid compatibility issues.</p>",R,A,_='<p>The following trainers currently support generation with vLLM:</p> <ul><li><a href="/docs/trl/pr_5607/en/gspo_token#trl.GRPOTrainer">GRPOTrainer</a></li> <li><a href="/docs/trl/pr_5607/en/rloo_trainer#trl.RLOOTrainer">RLOOTrainer</a></li> <li><a href="/docs/trl/pr_5607/en/nash_md_trainer#trl.experimental.nash_md.NashMDTrainer">experimental.nash_md.NashMDTrainer</a></li> <li><a href="/docs/trl/pr_5607/en/online_dpo_trainer#trl.experimental.online_dpo.OnlineDPOTrainer">experimental.online_dpo.OnlineDPOTrainer</a></li> <li><a href="/docs/trl/pr_5607/en/xpo_trainer#trl.experimental.xpo.XPOTrainer">experimental.xpo.XPOTrainer</a></li></ul>',E,x,We,H,Sl="💡 <strong>Note</strong>: Resources required for this specific example: a single node with 8 GPUs.",Xe,L,Fl=`<p>When using vLLM with TRL, the <strong>vLLM server</strong> and the <strong>trainer</strong> must run on <strong>separate CUDA devices</strong> to prevent conflicts.
For guidance on configuring this properly, see <a href="#modes-of-using-vllm-during-training">Modes of using vLLM during training</a>.</p>`,Se,Y,Ql="First, install vLLM using the following command:",Fe,O,Qe,z,Nl="Then run the server on specific GPUs (e.g., GPUs 0-3):",Ne,P,xe,D,xl="Once the server is running, you can use it to generate completions for training. In the example below, we are using the different supported trainers using the vLLM server for generation. The <code>--tensor-parallel-size</code> and <code>--data-parallel-size</code> arguments control how the model and data are sharded across GPUs.",He,q,Hl="In this example, we shard one model across 4 GPUs with tensor parallelism. Then, run the training script on different GPUs (e.g., GPUs 4-7) by passing <code>use_vllm=True</code> in the training arguments as follows:",Ye,K,Yl="Sample of a simple <code>train.py</code> script:",Oe,k,ze,ee,Ol="And the train command on separate GPUs from the server:",Pe,le,De,te,qe,se,Ke,ne,zl='Online methods like GRPO or Online DPO require the model to generate completions during training, which are then used to compute reward signals. However, generation can be extremely time-consuming, especially with large or reasoning models. In the default setup (without vLLM), completions are generated using the <a href="https://github.com/huggingface/trl/blob/f3e8c2304428ef16e9ae5de9e5741ed84d533b7b/trl/trainer/grpo_trainer.py#L965C39-L965C66" rel="nofollow">(unwrapped) model’s <code>generate</code> method</a>. This approach quickly becomes a major bottleneck — generation is slow and inefficient, particularly for large batches or models. As a result, training times increase significantly, and overall efficiency drops. To address this, we turn to vLLM, which enables much faster and more scalable generation, helping eliminate this bottleneck in online methods.',el,ae,ll,Me,Pl='If you’ve ever done autoregressive decoder training, you know all the input tokens to the LLM produce their attention key and value tensors, and these tensors are kept in GPU memory to later generate subsequent tokens based on them. These cached key and value tensors are often referred to as the KV cache. However, storing the KV cache occupies a lot of memory, so vLLM uses a technique called <strong>PagedAttention</strong> to solve this problem. PagedAttention, which is inspired by the OS’s virtual memory concept, stores continuous keys and values in <strong>non-contiguous memory space</strong>, which is much more efficient. The details of this are beyond the scope of this document, but in short, it allows the model to store the keys and values in a more efficient way, reducing the memory footprint and speeding up the generation process. If you are interested, make sure to check out the <a href="https://blog.vllm.ai/2023/06/20/vllm.html" rel="nofollow">vLLM PagedAttention</a> for more details.',tl,re,sl,oe,nl,ie,Dl="When you run for example",al,me,Ml,pe,ql=`<li><p>vLLM first spawns multiple workers to handle incoming requests in parallel. The number of workers is determined by multiplying the <code>--tensor-parallel-size</code> and <code>--data-parallel-size</code> values. In this example, it spawns 4 workers (4 × 1).
Each worker operates independently and processes a chunk of the incoming requests — which are basically the prompts sent to the server for generation.</p></li> <li><p>Once the incoming requests (prompts) are distributed across the workers, the model starts generating completions. Internally, the model’s weights are split across multiple GPUs based on the <code>--tensor-parallel-size</code> argument — this is how tensor parallelism is handled.</p></li> <li><p>Although the GPUs process requests independently and in parallel, they still need to communicate with each other. Remember that each GPU handles only a slice of the incoming prompts (for example, with 4 GPUs and 8 prompts using <code>--tensor-parallel-size=4</code>, each GPU participates in serving the full model).
This GPU-to-GPU communication is managed efficiently by NVIDIA’s NCCL library. The communication mainly ensures that each GPU gets its correct portion of the incoming requests — it’s lightweight and doesn’t interfere with generation itself.
Separately, the number of completions to generate per prompt is controlled by the <code>num_generations</code> setting in the GRPO config. For instance, if you set <code>num_generations=2</code> (like in the picture above), each prompt will have 2 completions. So, with 8 prompts and <code>num_generations=2</code>, you would end up with 16 completions total — regardless of the number of GPUs or parallelism settings.</p></li>`,rl,we,ol,de,Kl='<li>The vLLM server starts by running the command: <code>trl vllm-serve --model Qwen/Qwen2.5-7B</code>.</li> <li>Once the server is running, it generates completions based on requests from the client (trainer) using <code>vllm_client.generate</code> <a href="https://github.com/huggingface/trl/blob/cc044e35b285be7dc062764b3364e1e684db4c7c/trl/trainer/grpo_trainer.py#L1025-L1035" rel="nofollow">these lines</a>.</li> <li>The client (trainer) then requests these completions from the server.</li> <li>These completions are used to compute the reward signal.</li> <li>Based on the reward signal and the model’s output, the loss is computed, and the backward pass is performed to update the model’s weights.</li> <li><strong>Note</strong>: The server only handles completion generation — it doesn’t train the model. Therefore, the model’s weights aren’t updated on the server. Once the backward pass is complete, the client sends the updated weights to the server using <code>vllm_client.update_named_param(name, param.data)</code>.</li>',il,Te,et="When using vLLM, ensure the GPUs assigned for training and generation are separate to avoid NCCL communication conflicts. If you do not set the <code>CUDA_VISIBLE_DEVICES</code> environment variable, the training script will use all available GPUs by default, which may lead to device conflicts. Starting from TRL next release after v0.19.1, the code automatically detects and prevents same-device usage, raising a error at the vllm server process:",ml,ye,pl,Je,lt="For example, if you want to use GPUs 4–7 for training while the server runs on GPUs 0-3, set:",wl,ce,dl,Ue,Tl,ue,yl,he,tt="You can customize the server configuration by passing additional arguments.",Jl,fe,cl,je,Ul,Ie,st='<img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/tp_dp_throughput_8_gpus.png" alt="tp dp throughput 8 gpus"/> <img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/tp_dp_throughput_4_gpus.png" alt="tp dp throughput 4 gpus"/>',ul,W,nt='<p>The benchmark plots above were collected with older vLLM versions. Starting with <a href="https://github.com/vllm-project/vllm/pull/30739" rel="nofollow">vLLM PR #30739</a> (released in <code>0.14.0</code>), offline data parallel scaling for non-MoE (dense) models is no longer supported. To follow the latest recommendations, do not scale DP for non-MoE models.</p>',hl,$e,fl,ge,at=`vLLM can use the <strong>Transformers backend</strong> for model implementations, which works for both LLMs and VLMs.
To enable this, set <code>vllm_model_impl=&quot;transformers&quot;</code> in your configuration or pass it via the command-line argument.`,jl,be,Mt='For more details, check out <a href="https://blog.vllm.ai/2025/04/11/transformers-backend.html" rel="nofollow">vLLM Transformers Backend</a>.',Il,Ce,rt="Example:",$l,Ze,gl,Ae,bl,ve,ot="TRL supports <strong>two modes</strong> for integrating vLLM during training: <strong>colocate mode</strong> (default) and <strong>server mode</strong>.",Cl,Ge,Zl,Ve,it=`In <strong>colocate mode</strong>, vLLM runs inside the trainer process and shares GPU memory with the training model.
This avoids launching a separate server and can improve GPU utilization, but may lead to memory contention on the training GPUs. This is the default mode.`,Al,Re,mt="Example configuration:",vl,X,Gl,Ee,Vl,Be,pt=`In <strong>server mode</strong>, vLLM runs as a separate process on dedicated GPUs and communicates with the trainer via HTTP.
This setup is ideal if you have GPUs dedicated to inference.`,Rl,_e,wt="Example configuration:",El,S,Bl,F,dt="<p>Check the documentation of the trainer you are using for specific details on vLLM usage and parameters.</p>",_l,Q,Tt='<p>To reduce GPU memory usage when running vLLM, consider <a href="reducing_memory_usage#vllm-sleep-mode">enabling vLLM sleep mode</a>.</p>',Ll,Le,kl,ke,Wl;return u=new gt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),h=new v({props:{title:"vLLM Integration",local:"vllm-integration",headingTag:"h1"}}),x=new v({props:{title:"🚀 How can I use vLLM with TRL to speed up training?",local:"-how-can-i-use-vllm-with-trl-to-speed-up-training",headingTag:"h2"}}),O=new g({props:{code:"cGlwJTIwaW5zdGFsbCUyMCUyMnRybCU1QnZsbG0lNUQlMjI=",highlighted:'pip install <span class="hljs-string">&quot;trl[vllm]&quot;</span>',wrap:!1}}),P=new g({props:{code:"Q1VEQV9WSVNJQkxFX0RFVklDRVMlM0QwJTJDMSUyQzIlMkMzJTIwdHJsJTIwdmxsbS1zZXJ2ZSUyMC0tbW9kZWwlMjBRd2VuJTJGUXdlbjIuNS03QiUyMC0tdGVuc29yLXBhcmFsbGVsLXNpemUlMjA0",highlighted:"CUDA_VISIBLE_DEVICES=0,1,2,3 trl vllm-serve --model Qwen/Qwen2.5-7B --tensor-parallel-size 4",wrap:!1}}),k=new Xl({props:{id:"vllm examples",options:["GRPO","OnlineDPO","NashMD","XPO","RLOO"],$$slots:{default:[Vt]},$$scope:{ctx:U}}}),le=new g({props:{code:"Q1VEQV9WSVNJQkxFX0RFVklDRVMlM0Q0JTJDNSUyQzYlMkM3JTIwYWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluLnB5",highlighted:"CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch train.py",wrap:!1}}),te=new v({props:{title:"Why using vLLM?",local:"why-using-vllm",headingTag:"h2"}}),se=new v({props:{title:"🎬 Flashback: Why do we need to use vLLM in online methods?",local:"-flashback-why-do-we-need-to-use-vllm-in-online-methods",headingTag:"h3"}}),ae=new v({props:{title:"🤔 How does vLLM solve the slow generation issue?",local:"-how-does-vllm-solve-the-slow-generation-issue",headingTag:"h3"}}),re=new v({props:{title:"How vLLM Works (Under the Hood) 🔍",local:"how-vllm-works-under-the-hood-",headingTag:"h2"}}),oe=new v({props:{title:"🤔 What exactly happens when you run trl vllm-serve --model <model_name> ?",local:"-what-exactly-happens-when-you-run-trl-vllm-serve---model-ltmodelnamegt-",headingTag:"h3"}}),me=new g({props:{code:"Q1VEQV9WSVNJQkxFX0RFVklDRVMlM0QwJTJDMSUyQzIlMkMzJTIwdHJsJTIwdmxsbS1zZXJ2ZSUyMC0tbW9kZWwlMjBRd2VuJTJGUXdlbjIuNS03QiUyMC0tdGVuc29yLXBhcmFsbGVsLXNpemUlMjA0",highlighted:"CUDA_VISIBLE_DEVICES=0,1,2,3 trl vllm-serve --model Qwen/Qwen2.5-7B --tensor-parallel-size 4",wrap:!1}}),we=new v({props:{title:"🥸 More detail on what happens under the hood when running the server",local:"-more-detail-on-what-happens-under-the-hood-when-running-the-server",headingTag:"h3"}}),ye=new g({props:{code:"UnVudGltZUVycm9yJTNBJTIwQXR0ZW1wdGluZyUyMHRvJTIwdXNlJTIwdGhlJTIwc2FtZSUyMENVREElMjBkZXZpY2UlMjBmb3IlMjBtdWx0aXBsZSUyMGRpc3RpbmN0JTIwcm9sZXMlMkZyYW5rcyUyMHdpdGhpbiUyMHRoZSUyMHNhbWUlMjBjb21tdW5pY2F0b3IuJTIwJTBBRW5zdXJlJTIwdGhhdCUyMHRyYWluZXIlMjBpcyUyMHVzaW5nJTIwZGlmZmVyZW50JTIwZGV2aWNlcyUyMHRoYW4lMjB2TExNJTIwc2VydmVyLg==",highlighted:`RuntimeError: Attempting <span class="hljs-keyword">to</span> use the same CUDA device <span class="hljs-keyword">for</span> multiple <span class="hljs-keyword">distinct</span> roles/ranks <span class="hljs-keyword">within</span> the same communicator.
Ensure that trainer <span class="hljs-keyword">is</span> <span class="hljs-keyword">using</span> different devices than vLLM <span class="hljs-keyword">server</span>.`,wrap:!1}}),ce=new g({props:{code:"Q1VEQV9WSVNJQkxFX0RFVklDRVMlM0Q0JTJDNSUyQzYlMkM3JTIwYWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluLnB5",highlighted:"CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch train.py",wrap:!1}}),Ue=new v({props:{title:"Advanced usage",local:"advanced-usage",headingTag:"h2"}}),ue=new v({props:{title:"🍷 More customization options with vLLM?",local:"-more-customization-options-with-vllm",headingTag:"h3"}}),fe=new g({props:{code:"JTI0JTIwdHJsJTIwdmxsbS1zZXJ2ZSUyMC0taGVscCUwQXVzYWdlJTNBJTIwdHJsJTIwdmxsbS1zZXJ2ZSUyMCU1Qi1oJTVEJTIwLS1tb2RlbCUyME1PREVMJTIwJTVCLS1yZXZpc2lvbiUyMFJFVklTSU9OJTVEJTIwJTVCLS10ZW5zb3JfcGFyYWxsZWxfc2l6ZSUyMFRFTlNPUl9QQVJBTExFTF9TSVpFJTVEJTIwJTVCLS1kYXRhX3BhcmFsbGVsX3NpemUlMjBEQVRBX1BBUkFMTEVMX1NJWkUlNUQlMjAlNUItLWhvc3QlMjBIT1NUJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCLS1wb3J0JTIwUE9SVCU1RCUyMCU1Qi0tZ3B1X21lbW9yeV91dGlsaXphdGlvbiUyMEdQVV9NRU1PUllfVVRJTElaQVRJT04lNUQlMjAlNUItLWR0eXBlJTIwRFRZUEUlNUQlMjAlNUItLW1heF9tb2RlbF9sZW4lMjBNQVhfTU9ERUxfTEVOJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCLS1lbmFibGVfcHJlZml4X2NhY2hpbmclMjBFTkFCTEVfUFJFRklYX0NBQ0hJTkclNUQlMjAlNUItLWVuZm9yY2VfZWFnZXIlMjAlNUJFTkZPUkNFX0VBR0VSJTVEJTVEJTIwJTVCLS1rdl9jYWNoZV9kdHlwZSUyMEtWX0NBQ0hFX0RUWVBFJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCLS10cnVzdF9yZW1vdGVfY29kZSUyMCU1QlRSVVNUX1JFTU9URV9DT0RFJTVEJTVEJTIwJTVCLS1sb2dfbGV2ZWwlMjBMT0dfTEVWRUwlNUQlMjAlNUItLXZsbG1fbW9kZWxfaW1wbCUyMFZMTE1fTU9ERUxfSU1QTCU1RCUwQSUwQW9wdGlvbnMlM0ElMEElMjAlMjAtaCUyQyUyMC0taGVscCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNob3clMjB0aGlzJTIwaGVscCUyMG1lc3NhZ2UlMjBhbmQlMjBleGl0JTBBJTIwJTIwLS1tb2RlbCUyME1PREVMJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTW9kZWwlMjBuYW1lJTIwb3IlMjBwYXRoJTIwdG8lMjBsb2FkJTIwdGhlJTIwbW9kZWwlMjBmcm9tLiUyMChkZWZhdWx0JTNBJTIwTm9uZSklMEElMjAlMjAtLXJldmlzaW9uJTIwUkVWSVNJT04lMjAlMjAlMjBSZXZpc2lvbiUyMHRvJTIwdXNlJTIwZm9yJTIwdGhlJTIwbW9kZWwuJTIwSWYlMjBub3QlMjBzcGVjaWZpZWQlMkMlMjB0aGUlMjBkZWZhdWx0JTIwYnJhbmNoJTIwd2lsbCUyMGJlJTIwdXNlZC4lMjAoZGVmYXVsdCUzQSUyME5vbmUpJTBBJTIwJTIwLS10ZW5zb3JfcGFyYWxsZWxfc2l6ZSUyMFRFTlNPUl9QQVJBTExFTF9TSVpFJTJDJTIwLS10ZW5zb3ItcGFyYWxsZWwtc2l6ZSUyMFRFTlNPUl9QQVJBTExFTF9TSVpFJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTnVtYmVyJTIwb2YlMjB0ZW5zb3IlMjBwYXJhbGxlbCUyMHdvcmtlcnMlMjB0byUyMHVzZS4lMjAoZGVmYXVsdCUzQSUyMDEpJTBBJTIwJTIwLS1kYXRhX3BhcmFsbGVsX3NpemUlMjBEQVRBX1BBUkFMTEVMX1NJWkUlMkMlMjAtLWRhdGEtcGFyYWxsZWwtc2l6ZSUyMERBVEFfUEFSQUxMRUxfU0laRSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME51bWJlciUyMG9mJTIwZGF0YSUyMHBhcmFsbGVsJTIwd29ya2VycyUyMHRvJTIwdXNlLiUyMEZvciUyMGRlbnNlJTIwbW9kZWxzJTJDJTIwa2VlcCUyMHRoaXMlMjBhdCUyMDEuJTIwU3RhcnRpbmclMjBmcm9tJTIwdkxMTSUyMCU2MDAuMTQuMCU2MCUyQyUyMHNldHRpbmclMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0aGlzJTIwYWJvdmUlMjAlNjAxJTYwJTIwZm9yJTIwZGVuc2UlMjBtb2RlbHMlMjBpcyUyMG5vJTIwbG9uZ2VyJTIwc3VwcG9ydGVkJTJGdXNlZnVsJTIwYW5kJTIwd2lsbCUyMGVycm9yJTIwb3V0JTIwKHNlZSUyMHZMTE0lMjBQUiUyMCUyMzMwNzM5KS4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAoZGVmYXVsdCUzQSUyMDEpJTBBJTIwJTIwLS1ob3N0JTIwSE9TVCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEhvc3QlMjBhZGRyZXNzJTIwdG8lMjBydW4lMjB0aGUlMjBzZXJ2ZXIlMjBvbi4lMjAoZGVmYXVsdCUzQSUyMDAuMC4wLjApJTBBJTIwJTIwLS1wb3J0JTIwUE9SVCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFBvcnQlMjB0byUyMHJ1biUyMHRoZSUyMHNlcnZlciUyMG9uLiUyMChkZWZhdWx0JTNBJTIwODAwMCklMEElMjAlMjAtLWdwdV9tZW1vcnlfdXRpbGl6YXRpb24lMjBHUFVfTUVNT1JZX1VUSUxJWkFUSU9OJTJDJTIwLS1ncHUtbWVtb3J5LXV0aWxpemF0aW9uJTIwR1BVX01FTU9SWV9VVElMSVpBVElPTiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFJhdGlvJTIwKGJldHdlZW4lMjAwJTIwYW5kJTIwMSklMjBvZiUyMEdQVSUyMG1lbW9yeSUyMHRvJTIwcmVzZXJ2ZSUyMGZvciUyMHRoZSUyMG1vZGVsJTIwd2VpZ2h0cyUyQyUyMGFjdGl2YXRpb25zJTJDJTIwYW5kJTIwS1YlMjBjYWNoZSUyMG9uJTIwdGhlJTIwZGV2aWNlJTIwZGVkaWNhdGVkJTIwdG8lMjBnZW5lcmF0aW9uJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcG93ZXJlZCUyMGJ5JTIwdkxMTS4lMjBIaWdoZXIlMjB2YWx1ZXMlMjB3aWxsJTIwaW5jcmVhc2UlMjB0aGUlMjBLViUyMGNhY2hlJTIwc2l6ZSUyMGFuZCUyMHRodXMlMjBpbXByb3ZlJTIwdGhlJTIwbW9kZWwncyUyMHRocm91Z2hwdXQuJTIwSG93ZXZlciUyQyUyMGlmJTIwdGhlJTIwdmFsdWUlMjBpcyUyMHRvbyUyMGhpZ2glMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpdCUyMG1heSUyMGNhdXNlJTIwb3V0LW9mLW1lbW9yeSUyMChPT00pJTIwZXJyb3JzJTIwZHVyaW5nJTIwaW5pdGlhbGl6YXRpb24uJTIwKGRlZmF1bHQlM0ElMjAwLjkpJTBBJTIwJTIwLS1kdHlwZSUyMERUWVBFJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwRGF0YSUyMHR5cGUlMjB0byUyMHVzZSUyMGZvciUyMHZMTE0lMjBnZW5lcmF0aW9uLiUyMElmJTIwc2V0JTIwdG8lMjAnYXV0byclMkMlMjB0aGUlMjBkYXRhJTIwdHlwZSUyMHdpbGwlMjBiZSUyMGF1dG9tYXRpY2FsbHklMjBkZXRlcm1pbmVkJTIwYmFzZWQlMjBvbiUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbi4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBGaW5kJTIwdGhlJTIwc3VwcG9ydGVkJTIwdmFsdWVzJTIwaW4lMjB0aGUlMjB2TExNJTIwZG9jdW1lbnRhdGlvbi4lMjAoZGVmYXVsdCUzQSUyMGF1dG8pJTBBJTIwJTIwLS1tYXhfbW9kZWxfbGVuJTIwTUFYX01PREVMX0xFTiUyQyUyMC0tbWF4LW1vZGVsLWxlbiUyME1BWF9NT0RFTF9MRU4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBJZiUyMHNldCUyQyUyMHRoZSUyMCU2MG1heF9tb2RlbF9sZW4lNjAlMjB0byUyMHVzZSUyMGZvciUyMHZMTE0uJTIwVGhpcyUyMGNhbiUyMGJlJTIwdXNlZnVsJTIwd2hlbiUyMHJ1bm5pbmclMjB3aXRoJTIwcmVkdWNlZCUyMCU2MHZsbG1fZ3B1X21lbW9yeV91dGlsaXphdGlvbiU2MCUyQyUyMGxlYWRpbmclMjB0byUyMGElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZWR1Y2VkJTIwS1YlMjBjYWNoZSUyMHNpemUuJTIwSWYlMjBub3QlMjBzZXQlMkMlMjB2TExNJTIwd2lsbCUyMHVzZSUyMHRoZSUyMG1vZGVsJTIwY29udGV4dCUyMHNpemUlMkMlMjB3aGljaCUyMG1pZ2h0JTIwYmUlMjBtdWNoJTIwbGFyZ2VyJTIwdGhhbiUyMHRoZSUyMEtWJTIwY2FjaGUlMkMlMjBsZWFkaW5nJTIwdG8lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbmVmZmljaWVuY2llcy4lMjAoZGVmYXVsdCUzQSUyME5vbmUpJTBBJTIwJTIwLS1lbmFibGVfcHJlZml4X2NhY2hpbmclMjBFTkFCTEVfUFJFRklYX0NBQ0hJTkclMkMlMjAtLWVuYWJsZS1wcmVmaXgtY2FjaGluZyUyMEVOQUJMRV9QUkVGSVhfQ0FDSElORyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFdoZXRoZXIlMjB0byUyMGVuYWJsZSUyMHByZWZpeCUyMGNhY2hpbmclMjBpbiUyMHZMTE0uJTIwSWYlMjBzZXQlMjB0byUyMCU2MFRydWUlNjAlMkMlMjBlbnN1cmUlMjB0aGF0JTIwdGhlJTIwbW9kZWwlMjBhbmQlMjB0aGUlMjBoYXJkd2FyZSUyMHN1cHBvcnQlMjB0aGlzJTIwZmVhdHVyZS4lMjAoZGVmYXVsdCUzQSUyME5vbmUpJTBBJTIwJTIwLS1lbmZvcmNlX2VhZ2VyJTIwJTVCRU5GT1JDRV9FQUdFUiU1RCUyQyUyMC0tZW5mb3JjZS1lYWdlciUyMCU1QkVORk9SQ0VfRUFHRVIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBXaGV0aGVyJTIwdG8lMjBlbmZvcmNlJTIwZWFnZXIlMjBleGVjdXRpb24uJTIwSWYlMjBzZXQlMjB0byUyMCU2MFRydWUlNjAlMkMlMjB3ZSUyMHdpbGwlMjBkaXNhYmxlJTIwQ1VEQSUyMGdyYXBoJTIwYW5kJTIwYWx3YXlzJTIwZXhlY3V0ZSUyMHRoZSUyMG1vZGVsJTIwaW4lMjBlYWdlciUyMG1vZGUuJTIwSWYlMjAlNjBGYWxzZSU2MCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMChkZWZhdWx0JTIwYmVoYXZpb3IpJTJDJTIwd2UlMjB3aWxsJTIwdXNlJTIwQ1VEQSUyMGdyYXBoJTIwYW5kJTIwZWFnZXIlMjBleGVjdXRpb24lMjBpbiUyMGh5YnJpZC4lMjAoZGVmYXVsdCUzQSUyMEZhbHNlKSUwQSUyMCUyMC0ta3ZfY2FjaGVfZHR5cGUlMjBLVl9DQUNIRV9EVFlQRSUyQyUyMC0ta3YtY2FjaGUtZHR5cGUlMjBLVl9DQUNIRV9EVFlQRSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMERhdGElMjB0eXBlJTIwdG8lMjB1c2UlMjBmb3IlMjBLViUyMGNhY2hlLiUyMElmJTIwc2V0JTIwdG8lMjAnYXV0byclMkMlMjB0aGUlMjBkdHlwZSUyMHdpbGwlMjBkZWZhdWx0JTIwdG8lMjB0aGUlMjBtb2RlbCUyMGRhdGElMjB0eXBlLiUyMChkZWZhdWx0JTNBJTIwYXV0byklMEElMjAlMjAtLXRydXN0X3JlbW90ZV9jb2RlJTIwJTVCVFJVU1RfUkVNT1RFX0NPREUlNUQlMkMlMjAtLXRydXN0LXJlbW90ZS1jb2RlJTIwJTVCVFJVU1RfUkVNT1RFX0NPREUlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBXaGV0aGVyJTIwdG8lMjB0cnVzdCUyMHJlbW90ZSUyMGNvZGUlMjB3aGVuJTIwbG9hZGluZyUyMG1vZGVscy4lMjBTZXQlMjB0byUyMFRydWUlMjB0byUyMGFsbG93JTIwZXhlY3V0aW5nJTIwY29kZSUyMGZyb20lMjBtb2RlbCUyMHJlcG9zaXRvcmllcy4lMjBUaGlzJTIwaXMlMjByZXF1aXJlZCUyMGZvciUyMHNvbWUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjdXN0b20lMjBtb2RlbHMlMjBidXQlMjBpbnRyb2R1Y2VzJTIwc2VjdXJpdHklMjByaXNrcy4lMjAoZGVmYXVsdCUzQSUyMEZhbHNlKSUwQSUyMCUyMC0tbG9nX2xldmVsJTIwTE9HX0xFVkVMJTJDJTIwLS1sb2ctbGV2ZWwlMjBMT0dfTEVWRUwlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBMb2clMjBsZXZlbCUyMGZvciUyMHV2aWNvcm4uJTIwUG9zc2libGUlMjBjaG9pY2VzJTNBJTIwJ2NyaXRpY2FsJyUyQyUyMCdlcnJvciclMkMlMjAnd2FybmluZyclMkMlMjAnaW5mbyclMkMlMjAnZGVidWcnJTJDJTIwJ3RyYWNlJy4lMjAoZGVmYXVsdCUzQSUyMGluZm8pJTBBJTIwJTIwLS12bGxtX21vZGVsX2ltcGwlMjBWTExNX01PREVMX0lNUEwlMkMlMjAtLXZsbG0tbW9kZWwtaW1wbCUyMFZMTE1fTU9ERUxfSU1QTCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME1vZGVsJTIwaW1wbGVtZW50YXRpb24lMjB0byUyMHVzZSUyMGZvciUyMHZMTE0uJTIwTXVzdCUyMGJlJTIwb25lJTIwb2YlMjAlNjB0cmFuc2Zvcm1lcnMlNjAlMjBvciUyMCU2MHZsbG0lNjAuJTIwJTYwdHJhbnNmb3JtZXJzJTYwJTNBJTIwVXNlJTIwdGhlJTIwJTYwdHJhbnNmb3JtZXJzJTYwJTIwYmFja2VuZCUyMGZvciUyMG1vZGVsJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW1wbGVtZW50YXRpb24uJTIwJTYwdmxsbSU2MCUzQSUyMFVzZSUyMHRoZSUyMCU2MHZsbG0lNjAlMjBsaWJyYXJ5JTIwZm9yJTIwbW9kZWwlMjBpbXBsZW1lbnRhdGlvbi4lMjAoZGVmYXVsdCUzQSUyMHZsbG0p",highlighted:`$ trl vllm-serve --help
usage: trl vllm-serve [-h] --model MODEL [--revision REVISION] [--tensor_parallel_size TENSOR_PARALLEL_SIZE] [--data_parallel_size DATA_PARALLEL_SIZE] [--host HOST]
 [--port PORT] [--gpu_memory_utilization GPU_MEMORY_UTILIZATION] [--dtype DTYPE] [--max_model_len MAX_MODEL_LEN]
 [--enable_prefix_caching ENABLE_PREFIX_CACHING] [--enforce_eager [ENFORCE_EAGER]] [--kv_cache_dtype KV_CACHE_DTYPE]
 [--trust_remote_code [TRUST_REMOTE_CODE]] [--log_level LOG_LEVEL] [--vllm_model_impl VLLM_MODEL_IMPL]
options:
 -h, --help show this help message and exit
 --model MODEL Model name or path to load the model from. (default: None)
 --revision REVISION Revision to use for the model. If not specified, the default branch will be used. (default: None)
 --tensor_parallel_size TENSOR_PARALLEL_SIZE, --tensor-parallel-size TENSOR_PARALLEL_SIZE
 Number of tensor parallel workers to use. (default: 1)
 --data_parallel_size DATA_PARALLEL_SIZE, --data-parallel-size DATA_PARALLEL_SIZE
 Number of data parallel workers to use. For dense models, keep this at 1. Starting from vLLM \`0.14.0\`, setting
 this above \`1\` for dense models is no longer supported/useful and will error out (see vLLM PR #30739).
 (default: 1)
 --host HOST Host address to run the server on. (default: 0.0.0.0)
 --port PORT Port to run the server on. (default: 8000)
 --gpu_memory_utilization GPU_MEMORY_UTILIZATION, --gpu-memory-utilization GPU_MEMORY_UTILIZATION
 Ratio (between 0 and 1) of GPU memory to reserve for the model weights, activations, and KV cache on the device dedicated to generation
 powered by vLLM. Higher values will increase the KV cache size and thus improve the model&#x27;s throughput. However, if the value is too high,
 it may cause out-of-memory (OOM) errors during initialization. (default: 0.9)
 --dtype DTYPE Data type to use for vLLM generation. If set to &#x27;auto&#x27;, the data type will be automatically determined based on the model configuration.
 Find the supported values in the vLLM documentation. (default: auto)
 --max_model_len MAX_MODEL_LEN, --max-model-len MAX_MODEL_LEN
 If set, the \`max_model_len\` to use for vLLM. This can be useful when running with reduced \`vllm_gpu_memory_utilization\`, leading to a
 reduced KV cache size. If not set, vLLM will use the model context size, which might be much larger than the KV cache, leading to
 inefficiencies. (default: None)
 --enable_prefix_caching ENABLE_PREFIX_CACHING, --enable-prefix-caching ENABLE_PREFIX_CACHING
 Whether to enable prefix caching in vLLM. If set to \`True\`, ensure that the model and the hardware support this feature. (default: None)
 --enforce_eager [ENFORCE_EAGER], --enforce-eager [ENFORCE_EAGER]
 Whether to enforce eager execution. If set to \`True\`, we will disable CUDA graph and always execute the model in eager mode. If \`False\`
 (default behavior), we will use CUDA graph and eager execution in hybrid. (default: False)
 --kv_cache_dtype KV_CACHE_DTYPE, --kv-cache-dtype KV_CACHE_DTYPE
 Data type to use for KV cache. If set to &#x27;auto&#x27;, the dtype will default to the model data type. (default: auto)
 --trust_remote_code [TRUST_REMOTE_CODE], --trust-remote-code [TRUST_REMOTE_CODE]
 Whether to trust remote code when loading models. Set to True to allow executing code from model repositories. This is required for some
 custom models but introduces security risks. (default: False)
 --log_level LOG_LEVEL, --log-level LOG_LEVEL
 Log level for uvicorn. Possible choices: &#x27;critical&#x27;, &#x27;error&#x27;, &#x27;warning&#x27;, &#x27;info&#x27;, &#x27;debug&#x27;, &#x27;trace&#x27;. (default: info)
 --vllm_model_impl VLLM_MODEL_IMPL, --vllm-model-impl VLLM_MODEL_IMPL
 Model implementation to use for vLLM. Must be one of \`transformers\` or \`vllm\`. \`transformers\`: Use the \`transformers\` backend for model
 implementation. \`vllm\`: Use the \`vllm\` library for model implementation. (default: vllm)`,wrap:!1}}),je=new v({props:{title:"💆🏻‍♀️ What’s the best distributed setup?",local:"-whats-the-best-distributed-setup",headingTag:"h3"}}),$e=new v({props:{title:"vLLM with Transformers Backend",local:"vllm-with-transformers-backend",headingTag:"h3"}}),Ze=new g({props:{code:"Q1VEQV9ERVZJQ0VfT1JERVIlM0RQQ0lfQlVTX0lEJTIwQ1VEQV9WSVNJQkxFX0RFVklDRVMlM0QwJTIwdHJsJTIwdmxsbS1zZXJ2ZSUyMC0tbW9kZWwlMjBRd2VuJTJGUXdlbiUwQTIuNS1WTC0zQi1JbnN0cnVjdCUyMC0tdGVuc29yLXBhcmFsbGVsLXNpemUlMjAxJTIwLS1wb3J0JTIwODAwMCUyMC0tZW5mb3JjZV9lYWdlciUyMC0tdmxsbV9tb2RlbF9pbXBsJTIwdHJhbnNmb3JtZXJz",highlighted:`CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0 trl vllm-serve --model Qwen/Qwen
2.5-VL-3B-Instruct --tensor-parallel-size 1 --port 8000 --enforce_eager --vllm_model_impl transformers`,wrap:!1}}),Ae=new v({props:{title:"Modes of Using vLLM During Training",local:"modes-of-using-vllm-during-training",headingTag:"h3"}}),Ge=new v({props:{title:"Colocate Mode",local:"colocate-mode",headingTag:"h4"}}),X=new Xl({props:{id:"vllm examples",options:["GRPO","OnlineDPO","NashMD","XPO","RLOO"],$$slots:{default:[kt]},$$scope:{ctx:U}}}),Ee=new v({props:{title:"Server Mode",local:"server-mode",headingTag:"h4"}}),S=new Xl({props:{id:"vllm examples",options:["GRPO","OnlineDPO","NashMD","XPO","RLOO"],$$slots:{default:[Nt]},$$scope:{ctx:U}}}),Le=new bt({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/vllm_integration.md"}}),{c(){t=j("meta"),r=o(),l=j("p"),c=o(),p(u.$$.fragment),b=o(),p(h.$$.fragment),C=o(),f=j("p"),f.textContent=Z,s=o(),m=j("blockquote"),m.innerHTML=B,R=o(),A=j("blockquote"),A.innerHTML=_,E=o(),p(x.$$.fragment),We=o(),H=j("p"),H.innerHTML=Sl,Xe=o(),L=j("blockquote"),L.innerHTML=Fl,Se=o(),Y=j("p"),Y.textContent=Ql,Fe=o(),p(O.$$.fragment),Qe=o(),z=j("p"),z.textContent=Nl,Ne=o(),p(P.$$.fragment),xe=o(),D=j("p"),D.innerHTML=xl,He=o(),q=j("p"),q.innerHTML=Hl,Ye=o(),K=j("p"),K.innerHTML=Yl,Oe=o(),p(k.$$.fragment),ze=o(),ee=j("p"),ee.textContent=Ol,Pe=o(),p(le.$$.fragment),De=o(),p(te.$$.fragment),qe=o(),p(se.$$.fragment),Ke=o(),ne=j("p"),ne.innerHTML=zl,el=o(),p(ae.$$.fragment),ll=o(),Me=j("p"),Me.innerHTML=Pl,tl=o(),p(re.$$.fragment),sl=o(),p(oe.$$.fragment),nl=o(),ie=j("p"),ie.textContent=Dl,al=o(),p(me.$$.fragment),Ml=o(),pe=j("ol"),pe.innerHTML=ql,rl=o(),p(we.$$.fragment),ol=o(),de=j("ul"),de.innerHTML=Kl,il=o(),Te=j("p"),Te.innerHTML=et,ml=o(),p(ye.$$.fragment),pl=o(),Je=j("p"),Je.textContent=lt,wl=o(),p(ce.$$.fragment),dl=o(),p(Ue.$$.fragment),Tl=o(),p(ue.$$.fragment),yl=o(),he=j("p"),he.textContent=tt,Jl=o(),p(fe.$$.fragment),cl=o(),p(je.$$.fragment),Ul=o(),Ie=j("p"),Ie.innerHTML=st,ul=o(),W=j("blockquote"),W.innerHTML=nt,hl=o(),p($e.$$.fragment),fl=o(),ge=j("p"),ge.innerHTML=at,jl=o(),be=j("p"),be.innerHTML=Mt,Il=o(),Ce=j("p"),Ce.textContent=rt,$l=o(),p(Ze.$$.fragment),gl=o(),p(Ae.$$.fragment),bl=o(),ve=j("p"),ve.innerHTML=ot,Cl=o(),p(Ge.$$.fragment),Zl=o(),Ve=j("p"),Ve.innerHTML=it,Al=o(),Re=j("p"),Re.textContent=mt,vl=o(),p(X.$$.fragment),Gl=o(),p(Ee.$$.fragment),Vl=o(),Be=j("p"),Be.innerHTML=pt,Rl=o(),_e=j("p"),_e.textContent=wt,El=o(),p(S.$$.fragment),Bl=o(),F=j("blockquote"),F.innerHTML=dt,_l=o(),Q=j("blockquote"),Q.innerHTML=Tt,Ll=o(),p(Le.$$.fragment),kl=o(),ke=j("p"),this.h()},l(e){const n=It("svelte-u9bgzb",document.head);t=I(n,"META",{name:!0,content:!0}),n.forEach(a),r=i(e),l=I(e,"P",{}),Ut(l).forEach(a),c=i(e),w(u.$$.fragment,e),b=i(e),w(h.$$.fragment,e),C=i(e),f=I(e,"P",{"data-svelte-h":!0}),$(f)!=="svelte-1lxfs56"&&(f.textContent=Z),s=i(e),m=I(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),$(m)!=="svelte-qklt84"&&(m.innerHTML=B),R=i(e),A=I(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),$(A)!=="svelte-kodqo2"&&(A.innerHTML=_),E=i(e),w(x.$$.fragment,e),We=i(e),H=I(e,"P",{"data-svelte-h":!0}),$(H)!=="svelte-chqfxa"&&(H.innerHTML=Sl),Xe=i(e),L=I(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),$(L)!=="svelte-1tk8a4i"&&(L.innerHTML=Fl),Se=i(e),Y=I(e,"P",{"data-svelte-h":!0}),$(Y)!=="svelte-dqsh1"&&(Y.textContent=Ql),Fe=i(e),w(O.$$.fragment,e),Qe=i(e),z=I(e,"P",{"data-svelte-h":!0}),$(z)!=="svelte-1lqk5aw"&&(z.textContent=Nl),Ne=i(e),w(P.$$.fragment,e),xe=i(e),D=I(e,"P",{"data-svelte-h":!0}),$(D)!=="svelte-1ikzuko"&&(D.innerHTML=xl),He=i(e),q=I(e,"P",{"data-svelte-h":!0}),$(q)!=="svelte-1ne4viv"&&(q.innerHTML=Hl),Ye=i(e),K=I(e,"P",{"data-svelte-h":!0}),$(K)!=="svelte-1ui7abz"&&(K.innerHTML=Yl),Oe=i(e),w(k.$$.fragment,e),ze=i(e),ee=I(e,"P",{"data-svelte-h":!0}),$(ee)!=="svelte-v2qt3y"&&(ee.textContent=Ol),Pe=i(e),w(le.$$.fragment,e),De=i(e),w(te.$$.fragment,e),qe=i(e),w(se.$$.fragment,e),Ke=i(e),ne=I(e,"P",{"data-svelte-h":!0}),$(ne)!=="svelte-1falafi"&&(ne.innerHTML=zl),el=i(e),w(ae.$$.fragment,e),ll=i(e),Me=I(e,"P",{"data-svelte-h":!0}),$(Me)!=="svelte-1mphyvc"&&(Me.innerHTML=Pl),tl=i(e),w(re.$$.fragment,e),sl=i(e),w(oe.$$.fragment,e),nl=i(e),ie=I(e,"P",{"data-svelte-h":!0}),$(ie)!=="svelte-1m36fdl"&&(ie.textContent=Dl),al=i(e),w(me.$$.fragment,e),Ml=i(e),pe=I(e,"OL",{"data-svelte-h":!0}),$(pe)!=="svelte-14355j0"&&(pe.innerHTML=ql),rl=i(e),w(we.$$.fragment,e),ol=i(e),de=I(e,"UL",{"data-svelte-h":!0}),$(de)!=="svelte-1sheerw"&&(de.innerHTML=Kl),il=i(e),Te=I(e,"P",{"data-svelte-h":!0}),$(Te)!=="svelte-1dy0xt"&&(Te.innerHTML=et),ml=i(e),w(ye.$$.fragment,e),pl=i(e),Je=I(e,"P",{"data-svelte-h":!0}),$(Je)!=="svelte-1m4w16a"&&(Je.textContent=lt),wl=i(e),w(ce.$$.fragment,e),dl=i(e),w(Ue.$$.fragment,e),Tl=i(e),w(ue.$$.fragment,e),yl=i(e),he=I(e,"P",{"data-svelte-h":!0}),$(he)!=="svelte-1vcl05h"&&(he.textContent=tt),Jl=i(e),w(fe.$$.fragment,e),cl=i(e),w(je.$$.fragment,e),Ul=i(e),Ie=I(e,"P",{"data-svelte-h":!0}),$(Ie)!=="svelte-1gwb02o"&&(Ie.innerHTML=st),ul=i(e),W=I(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),$(W)!=="svelte-gniqfq"&&(W.innerHTML=nt),hl=i(e),w($e.$$.fragment,e),fl=i(e),ge=I(e,"P",{"data-svelte-h":!0}),$(ge)!=="svelte-uxxpw4"&&(ge.innerHTML=at),jl=i(e),be=I(e,"P",{"data-svelte-h":!0}),$(be)!=="svelte-1h516ml"&&(be.innerHTML=Mt),Il=i(e),Ce=I(e,"P",{"data-svelte-h":!0}),$(Ce)!=="svelte-11lpom8"&&(Ce.textContent=rt),$l=i(e),w(Ze.$$.fragment,e),gl=i(e),w(Ae.$$.fragment,e),bl=i(e),ve=I(e,"P",{"data-svelte-h":!0}),$(ve)!=="svelte-blaqyg"&&(ve.innerHTML=ot),Cl=i(e),w(Ge.$$.fragment,e),Zl=i(e),Ve=I(e,"P",{"data-svelte-h":!0}),$(Ve)!=="svelte-a8whk5"&&(Ve.innerHTML=it),Al=i(e),Re=I(e,"P",{"data-svelte-h":!0}),$(Re)!=="svelte-oqo0p2"&&(Re.textContent=mt),vl=i(e),w(X.$$.fragment,e),Gl=i(e),w(Ee.$$.fragment,e),Vl=i(e),Be=I(e,"P",{"data-svelte-h":!0}),$(Be)!=="svelte-1aupp20"&&(Be.innerHTML=pt),Rl=i(e),_e=I(e,"P",{"data-svelte-h":!0}),$(_e)!=="svelte-oqo0p2"&&(_e.textContent=wt),El=i(e),w(S.$$.fragment,e),Bl=i(e),F=I(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),$(F)!=="svelte-fah6my"&&(F.innerHTML=dt),_l=i(e),Q=I(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),$(Q)!=="svelte-1nu51hh"&&(Q.innerHTML=Tt),Ll=i(e),w(Le.$$.fragment,e),kl=i(e),ke=I(e,"P",{}),Ut(ke).forEach(a),this.h()},h(){N(t,"name","hf:doc:metadata"),N(t,"content",Ht),N(m,"class","warning"),N(A,"class","tip"),N(L,"class","warning"),N(W,"class","warning"),N(F,"class","warning"),N(Q,"class","warning")},m(e,n){$t(document.head,t),M(e,r,n),M(e,l,n),M(e,c,n),d(u,e,n),M(e,b,n),d(h,e,n),M(e,C,n),M(e,f,n),M(e,s,n),M(e,m,n),M(e,R,n),M(e,A,n),M(e,E,n),d(x,e,n),M(e,We,n),M(e,H,n),M(e,Xe,n),M(e,L,n),M(e,Se,n),M(e,Y,n),M(e,Fe,n),d(O,e,n),M(e,Qe,n),M(e,z,n),M(e,Ne,n),d(P,e,n),M(e,xe,n),M(e,D,n),M(e,He,n),M(e,q,n),M(e,Ye,n),M(e,K,n),M(e,Oe,n),d(k,e,n),M(e,ze,n),M(e,ee,n),M(e,Pe,n),d(le,e,n),M(e,De,n),d(te,e,n),M(e,qe,n),d(se,e,n),M(e,Ke,n),M(e,ne,n),M(e,el,n),d(ae,e,n),M(e,ll,n),M(e,Me,n),M(e,tl,n),d(re,e,n),M(e,sl,n),d(oe,e,n),M(e,nl,n),M(e,ie,n),M(e,al,n),d(me,e,n),M(e,Ml,n),M(e,pe,n),M(e,rl,n),d(we,e,n),M(e,ol,n),M(e,de,n),M(e,il,n),M(e,Te,n),M(e,ml,n),d(ye,e,n),M(e,pl,n),M(e,Je,n),M(e,wl,n),d(ce,e,n),M(e,dl,n),d(Ue,e,n),M(e,Tl,n),d(ue,e,n),M(e,yl,n),M(e,he,n),M(e,Jl,n),d(fe,e,n),M(e,cl,n),d(je,e,n),M(e,Ul,n),M(e,Ie,n),M(e,ul,n),M(e,W,n),M(e,hl,n),d($e,e,n),M(e,fl,n),M(e,ge,n),M(e,jl,n),M(e,be,n),M(e,Il,n),M(e,Ce,n),M(e,$l,n),d(Ze,e,n),M(e,gl,n),d(Ae,e,n),M(e,bl,n),M(e,ve,n),M(e,Cl,n),d(Ge,e,n),M(e,Zl,n),M(e,Ve,n),M(e,Al,n),M(e,Re,n),M(e,vl,n),d(X,e,n),M(e,Gl,n),d(Ee,e,n),M(e,Vl,n),M(e,Be,n),M(e,Rl,n),M(e,_e,n),M(e,El,n),d(S,e,n),M(e,Bl,n),M(e,F,n),M(e,_l,n),M(e,Q,n),M(e,Ll,n),d(Le,e,n),M(e,kl,n),M(e,ke,n),Wl=!0},p(e,[n]){const yt={};n&2&&(yt.$$scope={dirty:n,ctx:e}),k.$set(yt);const Jt={};n&2&&(Jt.$$scope={dirty:n,ctx:e}),X.$set(Jt);const ct={};n&2&&(ct.$$scope={dirty:n,ctx:e}),S.$set(ct)},i(e){Wl||(T(u.$$.fragment,e),T(h.$$.fragment,e),T(x.$$.fragment,e),T(O.$$.fragment,e),T(P.$$.fragment,e),T(k.$$.fragment,e),T(le.$$.fragment,e),T(te.$$.fragment,e),T(se.$$.fragment,e),T(ae.$$.fragment,e),T(re.$$.fragment,e),T(oe.$$.fragment,e),T(me.$$.fragment,e),T(we.$$.fragment,e),T(ye.$$.fragment,e),T(ce.$$.fragment,e),T(Ue.$$.fragment,e),T(ue.$$.fragment,e),T(fe.$$.fragment,e),T(je.$$.fragment,e),T($e.$$.fragment,e),T(Ze.$$.fragment,e),T(Ae.$$.fragment,e),T(Ge.$$.fragment,e),T(X.$$.fragment,e),T(Ee.$$.fragment,e),T(S.$$.fragment,e),T(Le.$$.fragment,e),Wl=!0)},o(e){y(u.$$.fragment,e),y(h.$$.fragment,e),y(x.$$.fragment,e),y(O.$$.fragment,e),y(P.$$.fragment,e),y(k.$$.fragment,e),y(le.$$.fragment,e),y(te.$$.fragment,e),y(se.$$.fragment,e),y(ae.$$.fragment,e),y(re.$$.fragment,e),y(oe.$$.fragment,e),y(me.$$.fragment,e),y(we.$$.fragment,e),y(ye.$$.fragment,e),y(ce.$$.fragment,e),y(Ue.$$.fragment,e),y(ue.$$.fragment,e),y(fe.$$.fragment,e),y(je.$$.fragment,e),y($e.$$.fragment,e),y(Ze.$$.fragment,e),y(Ae.$$.fragment,e),y(Ge.$$.fragment,e),y(X.$$.fragment,e),y(Ee.$$.fragment,e),y(S.$$.fragment,e),y(Le.$$.fragment,e),Wl=!1},d(e){e&&(a(r),a(l),a(c),a(b),a(C),a(f),a(s),a(m),a(R),a(A),a(E),a(We),a(H),a(Xe),a(L),a(Se),a(Y),a(Fe),a(Qe),a(z),a(Ne),a(xe),a(D),a(He),a(q),a(Ye),a(K),a(Oe),a(ze),a(ee),a(Pe),a(De),a(qe),a(Ke),a(ne),a(el),a(ll),a(Me),a(tl),a(sl),a(nl),a(ie),a(al),a(Ml),a(pe),a(rl),a(ol),a(de),a(il),a(Te),a(ml),a(pl),a(Je),a(wl),a(dl),a(Tl),a(yl),a(he),a(Jl),a(cl),a(Ul),a(Ie),a(ul),a(W),a(hl),a(fl),a(ge),a(jl),a(be),a(Il),a(Ce),a($l),a(gl),a(bl),a(ve),a(Cl),a(Zl),a(Ve),a(Al),a(Re),a(vl),a(Gl),a(Vl),a(Be),a(Rl),a(_e),a(El),a(Bl),a(F),a(_l),a(Q),a(Ll),a(kl),a(ke)),a(t),J(u,e),J(h,e),J(x,e),J(O,e),J(P,e),J(k,e),J(le,e),J(te,e),J(se,e),J(ae,e),J(re,e),J(oe,e),J(me,e),J(we,e),J(ye,e),J(ce,e),J(Ue,e),J(ue,e),J(fe,e),J(je,e),J($e,e),J(Ze,e),J(Ae,e),J(Ge,e),J(X,e),J(Ee,e),J(S,e),J(Le,e)}}}const Ht='{"title":"vLLM Integration","local":"vllm-integration","sections":[{"title":"🚀 How can I use vLLM with TRL to speed up training?","local":"-how-can-i-use-vllm-with-trl-to-speed-up-training","sections":[],"depth":2},{"title":"Why using vLLM?","local":"why-using-vllm","sections":[{"title":"🎬 Flashback: Why do we need to use vLLM in online methods?","local":"-flashback-why-do-we-need-to-use-vllm-in-online-methods","sections":[],"depth":3},{"title":"🤔 How does vLLM solve the slow generation issue?","local":"-how-does-vllm-solve-the-slow-generation-issue","sections":[],"depth":3}],"depth":2},{"title":"How vLLM Works (Under the Hood) 🔍","local":"how-vllm-works-under-the-hood-","sections":[{"title":"🤔 What exactly happens when you run trl vllm-serve --model &lt;model_name&gt; ?","local":"-what-exactly-happens-when-you-run-trl-vllm-serve---model-ltmodelnamegt-","sections":[],"depth":3},{"title":"🥸 More detail on what happens under the hood when running the server","local":"-more-detail-on-what-happens-under-the-hood-when-running-the-server","sections":[],"depth":3}],"depth":2},{"title":"Advanced usage","local":"advanced-usage","sections":[{"title":"🍷 More customization options with vLLM?","local":"-more-customization-options-with-vllm","sections":[],"depth":3},{"title":"💆🏻‍♀️ What’s the best distributed setup?","local":"-whats-the-best-distributed-setup","sections":[],"depth":3},{"title":"vLLM with Transformers Backend","local":"vllm-with-transformers-backend","sections":[],"depth":3},{"title":"Modes of Using vLLM During Training","local":"modes-of-using-vllm-during-training","sections":[{"title":"Colocate Mode","local":"colocate-mode","sections":[],"depth":4},{"title":"Server Mode","local":"server-mode","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}';function Yt(U){return ht(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Kt extends ft{constructor(t){super(),jt(this,t,Yt,xt,ut,{})}}export{Kt as component};

Xet Storage Details

Size:
60.8 kB
·
Xet hash:
4a3b26b74c545ec289611f6c777d35adc7ab967eccebbe50bc8bd600feb1d5a8

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.