trl/pr_5607/en/_app/immutable/nodes/24.cfae981d.js

import{s as Bs,n as Js,o as Vs}from"../chunks/scheduler.7b731bd4.js";import{S as Ws,i as Zs,e as p,s as n,c as d,q as L,H as B,h as Fs,a as m,d as a,b as l,f as x,g,j as Z,r as S,u as J,k as j,l as e,m as o,n as u,t as v,o as f,p as _}from"../chunks/index.cc268345.js";import{C as Xs,H as Gs,E as Qs}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f0d99f98.js";import{D as ws}from"../chunks/Docstring.03f7b462.js";import{C as Ys}from"../chunks/CodeBlock.169a125f.js";function Ks(Cs){let b,F,V,X,$,Q,T,Y,M,Es='In the paper <a href="https://huggingface.co/papers/2507.18071" rel="nofollow">Group Sequence Policy Optimization</a>, the authors propose a token-level objective variant to GSPO, called GSPO-token. To use GSPO-token, you can use the <code>GRPOTrainer</code> class in <code>trl.experimental.gspo_token</code>.',K,P,ss,N,as,O,r,ks,ts,js='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mover accent="true"><msub><mi>A</mi><mrow><mi>i</mi><mo separator="true">,</mo><mi>t</mi></mrow></msub><mo>^</mo></mover></mrow><annotation encoding="application/x-tex"> \\hat{A_{i,t}} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.2329em;vertical-align:-0.2861em;"></span><span class="mord accent"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.9468em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">i</span><span class="mpunct mtight">,</span><span class="mord mathnormal mtight">t</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span></span><span style="top:-3.2523em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.25em;"><span class="mord">^</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span></span></span>',es,ns,Ds='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>t</mi></mrow><annotation encoding="application/x-tex"> t </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6151em;"></span><span class="mord mathnormal">t</span></span></span></span>',ls,rs,qs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>i</mi></mrow><annotation encoding="application/x-tex"> i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6595em;"></span><span class="mord mathnormal">i</span></span></span></span>',os,is,Is='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mover accent="true"><msub><mi>A</mi><mrow><mi>i</mi><mo separator="true">,</mo><mi>t</mi></mrow></msub><mo>^</mo></mover></mrow><annotation encoding="application/x-tex"> \\hat{A_{i,t}} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.2329em;vertical-align:-0.2861em;"></span><span class="mord accent"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.9468em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">i</span><span class="mpunct mtight">,</span><span class="mord mathnormal mtight">t</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span></span><span style="top:-3.2523em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.25em;"><span class="mord">^</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span></span></span>',ps,ms,Us='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>t</mi></mrow><annotation encoding="application/x-tex"> t </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6151em;"></span><span class="mord mathnormal">t</span></span></span></span>',cs,hs,Hs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mover accent="true"><msub><mi>A</mi><mrow><mi>i</mi><mo separator="true">,</mo><mi>t</mi></mrow></msub><mo>^</mo></mover><mo>=</mo><mover accent="true"><msub><mi>A</mi><mi>i</mi></msub><mo>^</mo></mover></mrow><annotation encoding="application/x-tex"> \\hat{A_{i,t}}=\\hat{A_{i}} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.2329em;vertical-align:-0.2861em;"></span><span class="mord accent"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.9468em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">i</span><span class="mpunct mtight">,</span><span class="mord mathnormal mtight">t</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span></span><span style="top:-3.2523em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.25em;"><span class="mord">^</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.0968em;vertical-align:-0.15em;"></span><span class="mord accent"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.9468em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">i</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span><span style="top:-3.2523em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.25em;"><span class="mord">^</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span>',ds,gs,G,us,c,C,xs,y,E,$s,D,Rs="Main training entry point.",Ts,h,R,Ms,q,As="Will save the model, so you can reload it using <code>from_pretrained()</code>.",Ps,I,zs="Will only save from the main process.",Ns,w,A,Os,U,Ls="Upload <code>self.model</code> and <code>self.processing_class</code> to the 🤗 model hub on the repo <code>self.args.hub_model_id</code>.",vs,z,fs,W,_s;return $=new Xs({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),T=new Gs({props:{title:"GSPO-token",local:"gspo-token",headingTag:"h1"}}),P=new Gs({props:{title:"Usage",local:"usage",headingTag:"h2"}}),N=new Ys({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwuZ3Nwb190b2tlbiUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTBBZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwR1JQT0NvbmZpZyglMEElMjAlMjAlMjAlMjBpbXBvcnRhbmNlX3NhbXBsaW5nX2xldmVsJTNEJTIyc2VxdWVuY2VfdG9rZW4lMjIlMkMlMEElMjAlMjAlMjAlMjAuLi4lMEEp",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.gspo_token <span class="hljs-keyword">import</span> GRPOTrainer
<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig

training_args = GRPOConfig(
    importance_sampling_level=<span class="hljs-string">&quot;sequence_token&quot;</span>,
    ...
)`,wrap:!1}}),G=new Gs({props:{title:"GRPOTrainer",local:"trl.GRPOTrainer",headingTag:"h2"}}),C=new ws({props:{name:"class trl.GRPOTrainer",anchor:"trl.GRPOTrainer",parameters:[{name:"model",val:": str | PreTrainedModel | PeftModel"},{name:"reward_funcs",val:": str | transformers.modeling_utils.PreTrainedModel | collections.abc.Callable[..., list[float | None]] | list[str | transformers.modeling_utils.PreTrainedModel | collections.abc.Callable[..., list[float | None]]]"},{name:"args",val:": trl.trainer.grpo_config.GRPOConfig | None = None"},{name:"train_dataset",val:": datasets.arrow_dataset.Dataset | datasets.iterable_dataset.IterableDataset | None = None"},{name:"eval_dataset",val:": datasets.arrow_dataset.Dataset | datasets.iterable_dataset.IterableDataset | dict[str, datasets.arrow_dataset.Dataset | datasets.iterable_dataset.IterableDataset] | None = None"},{name:"processing_class",val:": transformers.tokenization_utils_base.PreTrainedTokenizerBase | transformers.processing_utils.ProcessorMixin | None = None"},{name:"reward_processing_classes",val:": transformers.tokenization_utils_base.PreTrainedTokenizerBase | list[transformers.tokenization_utils_base.PreTrainedTokenizerBase] | None = None"},{name:"callbacks",val:": list[transformers.trainer_callback.TrainerCallback] | None = None"},{name:"optimizers",val:": tuple = (None, None)"},{name:"peft_config",val:": PeftConfig | None = None"},{name:"tools",val:": list[collections.abc.Callable] | None = None"},{name:"rollout_func",val:": collections.abc.Callable[[list[str], 'GRPOTrainer'], dict[str, typing.Any]] | None = None"},{name:"environment_factory",val:": collections.abc.Callable[[], trl.trainer.grpo_trainer._SupportsReset] | None = None"}],source:"https://github.com/huggingface/trl/blob/vr_5607/trl/experimental/gspo_token/grpo_trainer.py#L21"}}),E=new ws({props:{name:"train",anchor:"trl.GRPOTrainer.train",parameters:[{name:"resume_from_checkpoint",val:": str | bool | None = None"},{name:"trial",val:": optuna.Trial | dict[str, Any] | None = None"},{name:"ignore_keys_for_eval",val:": list[str] | None = None"}],parametersDescription:[{anchor:"trl.GRPOTrainer.train.resume_from_checkpoint",description:`<strong>resume_from_checkpoint</strong> (<code>str</code> or <code>bool</code>, <em>optional</em>) &#x2014;
If a <code>str</code>, local path to a saved checkpoint as saved by a previous instance of <code>Trainer</code>. If a
<code>bool</code> and equals <code>True</code>, load the last checkpoint in <em>args.output_dir</em> as saved by a previous instance
of <code>Trainer</code>. If present, training will resume from the model/optimizer/scheduler states loaded here.`,name:"resume_from_checkpoint"},{anchor:"trl.GRPOTrainer.train.trial",description:`<strong>trial</strong> (<code>optuna.Trial</code> or <code>dict[str, Any]</code>, <em>optional</em>) &#x2014;
The trial run or the hyperparameter dictionary for hyperparameter search.`,name:"trial"},{anchor:"trl.GRPOTrainer.train.ignore_keys_for_eval",description:`<strong>ignore_keys_for_eval</strong> (<code>list[str]</code>, <em>optional</em>) &#x2014;
A list of keys in the output of your model (if it is a dictionary) that should be ignored when
gathering predictions for evaluation during the training.`,name:"ignore_keys_for_eval"}],source:"https://github.com/huggingface/trl/blob/vr_5607/transformers/trainer.py#L1323",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Object containing the global step count, training loss, and metrics.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>~trainer_utils.TrainOutput</code></p>
`}}),R=new ws({props:{name:"save_model",anchor:"trl.GRPOTrainer.save_model",parameters:[{name:"output_dir",val:": str | None = None"},{name:"_internal_call",val:": bool = False"}],source:"https://github.com/huggingface/trl/blob/vr_5607/transformers/trainer.py#L3746"}}),A=new ws({props:{name:"push_to_hub",anchor:"trl.GRPOTrainer.push_to_hub",parameters:[{name:"commit_message",val:": str | None = 'End of training'"},{name:"blocking",val:": bool = True"},{name:"token",val:": str | None = None"},{name:"revision",val:": str | None = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"trl.GRPOTrainer.push_to_hub.commit_message",description:`<strong>commit_message</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;End of training&quot;</code>) &#x2014;
Message to commit while pushing.`,name:"commit_message"},{anchor:"trl.GRPOTrainer.push_to_hub.blocking",description:`<strong>blocking</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether the function should return only when the <code>git push</code> has finished.`,name:"blocking"},{anchor:"trl.GRPOTrainer.push_to_hub.token",description:`<strong>token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Token with write permission to overwrite Trainer&#x2019;s original args.`,name:"token"},{anchor:"trl.GRPOTrainer.push_to_hub.revision",description:`<strong>revision</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The git revision to commit from. Defaults to the head of the &#x201C;main&#x201D; branch.`,name:"revision"},{anchor:"trl.GRPOTrainer.push_to_hub.kwargs",description:`<strong>kwargs</strong> (<code>dict[str, Any]</code>, <em>optional</em>) &#x2014;
Additional keyword arguments passed along to <code>~Trainer.create_model_card</code>.`,name:"kwargs"}],source:"https://github.com/huggingface/trl/blob/vr_5607/transformers/trainer.py#L3993",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The URL of the repository where the model was pushed if <code>blocking=False</code>, or a <code>Future</code> object tracking the
progress of the commit if <code>blocking=True</code>.</p>
`}}),z=new Qs({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/gspo_token.md"}}),{c(){b=p("meta"),F=n(),V=p("p"),X=n(),d($.$$.fragment),Q=n(),d(T.$$.fragment),Y=n(),M=p("p"),M.innerHTML=Es,K=n(),d(P.$$.fragment),ss=n(),d(N.$$.fragment),as=n(),O=p("blockquote"),r=p("p"),ks=L("To leverage GSPO-token, the user will need to provide the per-token advantage "),ts=new B(!1),es=L(" for each token "),ns=new B(!1),ls=L(" in the sequence "),rs=new B(!1),os=L(" (i.e., make "),is=new B(!1),ps=L(" varies with "),ms=new B(!1),cs=L("—which isn’t the case here, "),hs=new B(!1),ds=L("). Otherwise, GSPO-Token gradient is just equivalent to the original GSPO implementation."),gs=n(),d(G.$$.fragment),us=n(),c=p("div"),d(C.$$.fragment),xs=n(),y=p("div"),d(E.$$.fragment),$s=n(),D=p("p"),D.textContent=Rs,Ts=n(),h=p("div"),d(R.$$.fragment),Ms=n(),q=p("p"),q.innerHTML=As,Ps=n(),I=p("p"),I.textContent=zs,Ns=n(),w=p("div"),d(A.$$.fragment),Os=n(),U=p("p"),U.innerHTML=Ls,vs=n(),d(z.$$.fragment),fs=n(),W=p("p"),this.h()},l(s){const t=Fs("svelte-u9bgzb",document.head);b=m(t,"META",{name:!0,content:!0}),t.forEach(a),F=l(s),V=m(s,"P",{}),x(V).forEach(a),X=l(s),g($.$$.fragment,s),Q=l(s),g(T.$$.fragment,s),Y=l(s),M=m(s,"P",{"data-svelte-h":!0}),Z(M)!=="svelte-npbvk6"&&(M.innerHTML=Es),K=l(s),g(P.$$.fragment,s),ss=l(s),g(N.$$.fragment,s),as=l(s),O=m(s,"BLOCKQUOTE",{class:!0});var Ss=x(O);r=m(Ss,"P",{});var i=x(r);ks=S(i,"To leverage GSPO-token, the user will need to provide the per-token advantage "),ts=J(i,!1),es=S(i," for each token "),ns=J(i,!1),ls=S(i," in the sequence "),rs=J(i,!1),os=S(i," (i.e., make "),is=J(i,!1),ps=S(i," varies with "),ms=J(i,!1),cs=S(i,"—which isn’t the case here, "),hs=J(i,!1),ds=S(i,"). Otherwise, GSPO-Token gradient is just equivalent to the original GSPO implementation."),i.forEach(a),Ss.forEach(a),gs=l(s),g(G.$$.fragment,s),us=l(s),c=m(s,"DIV",{class:!0});var k=x(c);g(C.$$.fragment,k),xs=l(k),y=m(k,"DIV",{class:!0});var bs=x(y);g(E.$$.fragment,bs),$s=l(bs),D=m(bs,"P",{"data-svelte-h":!0}),Z(D)!=="svelte-1cilnet"&&(D.textContent=Rs),bs.forEach(a),Ts=l(k),h=m(k,"DIV",{class:!0});var H=x(h);g(R.$$.fragment,H),Ms=l(H),q=m(H,"P",{"data-svelte-h":!0}),Z(q)!=="svelte-r8h4ov"&&(q.innerHTML=As),Ps=l(H),I=m(H,"P",{"data-svelte-h":!0}),Z(I)!=="svelte-1e6bius"&&(I.textContent=zs),H.forEach(a),Ns=l(k),w=m(k,"DIV",{class:!0});var ys=x(w);g(A.$$.fragment,ys),Os=l(ys),U=m(ys,"P",{"data-svelte-h":!0}),Z(U)!=="svelte-8tudwd"&&(U.innerHTML=Ls),ys.forEach(a),k.forEach(a),vs=l(s),g(z.$$.fragment,s),fs=l(s),W=m(s,"P",{}),x(W).forEach(a),this.h()},h(){j(b,"name","hf:doc:metadata"),j(b,"content",sa),ts.a=es,ns.a=ls,rs.a=os,is.a=ps,ms.a=cs,hs.a=ds,j(O,"class","warning"),j(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(h,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(c,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(s,t){e(document.head,b),o(s,F,t),o(s,V,t),o(s,X,t),u($,s,t),o(s,Q,t),u(T,s,t),o(s,Y,t),o(s,M,t),o(s,K,t),u(P,s,t),o(s,ss,t),u(N,s,t),o(s,as,t),o(s,O,t),e(O,r),e(r,ks),ts.m(js,r),e(r,es),ns.m(Ds,r),e(r,ls),rs.m(qs,r),e(r,os),is.m(Is,r),e(r,ps),ms.m(Us,r),e(r,cs),hs.m(Hs,r),e(r,ds),o(s,gs,t),u(G,s,t),o(s,us,t),o(s,c,t),u(C,c,null),e(c,xs),e(c,y),u(E,y,null),e(y,$s),e(y,D),e(c,Ts),e(c,h),u(R,h,null),e(h,Ms),e(h,q),e(h,Ps),e(h,I),e(c,Ns),e(c,w),u(A,w,null),e(w,Os),e(w,U),o(s,vs,t),u(z,s,t),o(s,fs,t),o(s,W,t),_s=!0},p:Js,i(s){_s||(v($.$$.fragment,s),v(T.$$.fragment,s),v(P.$$.fragment,s),v(N.$$.fragment,s),v(G.$$.fragment,s),v(C.$$.fragment,s),v(E.$$.fragment,s),v(R.$$.fragment,s),v(A.$$.fragment,s),v(z.$$.fragment,s),_s=!0)},o(s){f($.$$.fragment,s),f(T.$$.fragment,s),f(P.$$.fragment,s),f(N.$$.fragment,s),f(G.$$.fragment,s),f(C.$$.fragment,s),f(E.$$.fragment,s),f(R.$$.fragment,s),f(A.$$.fragment,s),f(z.$$.fragment,s),_s=!1},d(s){s&&(a(F),a(V),a(X),a(Q),a(Y),a(M),a(K),a(ss),a(as),a(O),a(gs),a(us),a(c),a(vs),a(fs),a(W)),a(b),_($,s),_(T,s),_(P,s),_(N,s),_(G,s),_(C),_(E),_(R),_(A),_(z,s)}}}const sa='{"title":"GSPO-token","local":"gspo-token","sections":[{"title":"Usage","local":"usage","sections":[],"depth":2},{"title":"GRPOTrainer","local":"trl.GRPOTrainer","sections":[],"depth":2}],"depth":1}';function aa(Cs){return Vs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class oa extends Ws{constructor(b){super(),Zs(this,b,aa,Ks,Bs,{})}}export{oa as component};