trl/pr_5320/en/_app/immutable/nodes/4.a00a439c.js

import{s as $a,o as Ea,n as Ta}from"../chunks/scheduler.7b731bd4.js";import{S as za,i as Aa,e as p,s as n,c as u,q,H as T,h as Ca,a as m,d as t,b as l,f as z,g as f,j as P,r as k,u as A,k as Y,l as a,m as o,n as v,t as y,o as b,p as w}from"../chunks/index.cc268345.js";import{C as Ba,H as Ws,E as Da}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.12f929f6.js";import{D as is}from"../chunks/Docstring.9f263dc6.js";import{C as da}from"../chunks/CodeBlock.234f9e3d.js";import{E as Ja}from"../chunks/ExampleCodeBlock.b5d6e726.js";function Ua(rs){let h,F="Example:",C,x,M;return x=new da({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEJFTUFDYWxsYmFjayUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKC4uLiUyQyUyMGNhbGxiYWNrcyUzRCU1QkJFTUFDYWxsYmFjaygpJTVEKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> BEMACallback

trainer = Trainer(..., callbacks=[BEMACallback()])`,wrap:!1}}),{c(){h=p("p"),h.textContent=F,C=n(),u(x.$$.fragment)},l(i){h=m(i,"P",{"data-svelte-h":!0}),P(h)!=="svelte-11lpom8"&&(h.textContent=F),C=l(i),f(x.$$.fragment,i)},m(i,$){o(i,h,$),o(i,C,$),v(x,i,$),M=!0},p:Ta,i(i){M||(y(x.$$.fragment,i),M=!0)},o(i){b(x.$$.fragment,i),M=!1},d(i){i&&(t(h),t(C)),w(x,i)}}}function La(rs){let h,F,C,x,M,i,$,os,N,ma="This feature implements the BEMA algorithm to update the reference model during DPO training.",cs,W,hs,j,gs,I,ds,_,Z,js,J,Q,Is,S,ia="Main training entry point.",Zs,B,O,Qs,K,ra="Will save the model, so you can reload it using <code>from_pretrained()</code>.",Os,ss,oa="Will only save from the main process.",Gs,U,G,Vs,as,ca="Upload <code>self.model</code> and <code>self.processing_class</code> to the 🤗 model hub on the repo <code>self.args.hub_model_id</code>.",us,V,fs,r,R,Rs,ts,ha=`A <a href="https://huggingface.co/docs/transformers/main/en/main_classes/callback#transformers.TrainerCallback" rel="nofollow">TrainerCallback</a> that implements <a href="https://huggingface.co/papers/2508.00180" rel="nofollow">BEMA</a>
(Bias-Corrected Exponential Moving Average) by <a href="https://huggingface.co/abblock" rel="nofollow">Adam Block</a> and <a href="https://huggingface.co/cyrilzhang" rel="nofollow">Cyril
Zhang</a>. Code from <a href="https://github.com/abblock/bema" rel="nofollow">https://github.com/abblock/bema</a> under MIT license.`,Hs,es,Xs,vs,ua=`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msubsup><mi>θ</mi><mi>t</mi><mo mathvariant="normal" lspace="0em" rspace="0em">′</mo></msubsup><mo>=</mo><msub><mi>α</mi><mi>t</mi></msub><mo>⋅</mo><mo stretchy="false">(</mo><msub><mi>θ</mi><mi>t</mi></msub><mo>−</mo><msub><mi>θ</mi><mn>0</mn></msub><mo stretchy="false">)</mo><mo>+</mo><msub><mtext>EMA</mtext><mi>t</mi></msub></mrow><annotation encoding="application/x-tex">
\\theta_t&#x27; = \\alpha_t \\cdot (\\theta_t - \\theta_0) + \\text{EMA}_t
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.0489em;vertical-align:-0.247em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.8019em;"><span style="top:-2.453em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span><span style="top:-3.113em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">′</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.247em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.5945em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0037em;">α</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0037em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">⋅</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">0</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord text"><span class="mord">EMA</span></span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span></span>`,Ys,c,Ss,ys,fa='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>θ</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex"> \\theta_t </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',bs,ws,va='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>θ</mi><mn>0</mn></msub></mrow><annotation encoding="application/x-tex"> \\theta_0 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">0</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',_s,ns,ga="update_after",Ks,xs,ya='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mtext>EMA</mtext><mi>t</mi></msub></mrow><annotation encoding="application/x-tex"> \\text{EMA}_t  </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord text"><span class="mord">EMA</span></span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Ms,qs,ba='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>α</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex"> \\alpha_t </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0037em;">α</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0037em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',ks,$s,wa='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>t</mi></mrow><annotation encoding="application/x-tex"> t </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6151em;"></span><span class="mord mathnormal">t</span></span></span></span>',Es,Ts,_a=`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>α</mi><mi>t</mi></msub><mo>=</mo><mo stretchy="false">(</mo><mi>ρ</mi><mo>+</mo><mi>γ</mi><mo>⋅</mo><mi>t</mi><msup><mo stretchy="false">)</mo><mrow><mo>−</mo><mi>η</mi></mrow></msup><mi mathvariant="normal">.</mi></mrow><annotation encoding="application/x-tex">
\\alpha_t = (\\rho + \\gamma \\cdot t)^{-\\eta}.
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0037em;">α</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0037em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mord mathnormal">ρ</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.6389em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05556em;">γ</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">⋅</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.0713em;vertical-align:-0.25em;"></span><span class="mord mathnormal">t</span><span class="mclose"><span class="mclose">)</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.8213em;"><span style="top:-3.113em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">−</span><span class="mord mathnormal mtight" style="margin-right:0.03588em;">η</span></span></span></span></span></span></span></span></span><span class="mord">.</span></span></span></span></span>`,sa,ls,aa,zs,xa=`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mtext>EMA</mtext><mi>t</mi></msub><mo>=</mo><mo stretchy="false">(</mo><mn>1</mn><mo>−</mo><msub><mi>β</mi><mi>t</mi></msub><mo stretchy="false">)</mo><mo>⋅</mo><msub><mtext>EMA</mtext><mrow><mi>t</mi><mo>−</mo><mn>1</mn></mrow></msub><mo>+</mo><msub><mi>β</mi><mi>t</mi></msub><mo>⋅</mo><msub><mi>θ</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex">
\\text{EMA}_t = (1 - \\beta_t) \\cdot \\text{EMA}_{t-1} + \\beta_t \\cdot \\theta_t
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord text"><span class="mord">EMA</span></span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.05278em;">β</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0528em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">⋅</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8917em;vertical-align:-0.2083em;"></span><span class="mord"><span class="mord text"><span class="mord">EMA</span></span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">t</span><span class="mbin mtight">−</span><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2083em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.05278em;">β</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0528em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">⋅</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span></span>`,ta,D,ea,As,Ma='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>β</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex"> \\beta_t </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.05278em;">β</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0528em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Cs,Bs,qa='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>t</mi></mrow><annotation encoding="application/x-tex"> t </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6151em;"></span><span class="mord mathnormal">t</span></span></span></span>',Ds,Js,ka=`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>β</mi><mi>t</mi></msub><mo>=</mo><mo stretchy="false">(</mo><mi>ρ</mi><mo>+</mo><mi>γ</mi><mo>⋅</mo><mi>t</mi><msup><mo stretchy="false">)</mo><mrow><mo>−</mo><mi>κ</mi></mrow></msup><mi mathvariant="normal">.</mi></mrow><annotation encoding="application/x-tex">
\\beta_t = (\\rho + \\gamma \\cdot t)^{-\\kappa}.
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.05278em;">β</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:-0.0528em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mord mathnormal">ρ</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.6389em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05556em;">γ</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">⋅</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.0713em;vertical-align:-0.25em;"></span><span class="mord mathnormal">t</span><span class="mclose"><span class="mclose">)</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.8213em;"><span style="top:-3.113em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">−</span><span class="mord mathnormal mtight">κ</span></span></span></span></span></span></span></span></span><span class="mord">.</span></span></span></span></span>`,na,L,Us,H,Ls,ms,Ps;return M=new Ba({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),$=new Ws({props:{title:"BEMA for Reference Model",local:"bema-for-reference-model",headingTag:"h1"}}),W=new Ws({props:{title:"Usage",local:"usage",headingTag:"h2"}}),j=new da({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwuYmVtYV9mb3JfcmVmX21vZGVsJTIwaW1wb3J0JTIwQkVNQUNhbGxiYWNrJTJDJTIwRFBPVHJhaW5lciUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQSUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIydHJsLWludGVybmFsLXRlc3RpbmclMkZ6ZW4lMjIlMkMlMjAlMjJzdGFuZGFyZF9wcmVmZXJlbmNlJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEElMEFiZW1hX2NhbGxiYWNrJTIwJTNEJTIwQkVNQUNhbGxiYWNrKHVwZGF0ZV9yZWZfbW9kZWwlM0RUcnVlKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBEUE9UcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEJTIydHJsLWludGVybmFsLXRlc3RpbmclMkZ0aW55LVF3ZW4yRm9yQ2F1c2FsTE0tMi41JTIyJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBjYWxsYmFja3MlM0QlNUJiZW1hX2NhbGxiYWNrJTVEJTJDJTBBKSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.bema_for_ref_model <span class="hljs-keyword">import</span> BEMACallback, DPOTrainer
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

dataset = load_dataset(<span class="hljs-string">&quot;trl-internal-testing/zen&quot;</span>, <span class="hljs-string">&quot;standard_preference&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)

bema_callback = BEMACallback(update_ref_model=<span class="hljs-literal">True</span>)

trainer = DPOTrainer(
    model=<span class="hljs-string">&quot;trl-internal-testing/tiny-Qwen2ForCausalLM-2.5&quot;</span>,
    train_dataset=dataset,
    callbacks=[bema_callback],
)
trainer.train()`,wrap:!1}}),I=new Ws({props:{title:"DPOTrainer",local:"trl.DPOTrainer",headingTag:"h2"}}),Z=new is({props:{name:"class trl.DPOTrainer",anchor:"trl.DPOTrainer",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/trl/blob/vr_5320/trl/experimental/bema_for_ref_model/dpo_trainer.py#L19"}}),Q=new is({props:{name:"train",anchor:"trl.DPOTrainer.train",parameters:[{name:"resume_from_checkpoint",val:": str | bool | None = None"},{name:"trial",val:": optuna.Trial | dict[str, Any] | None = None"},{name:"ignore_keys_for_eval",val:": list[str] | None = None"}],parametersDescription:[{anchor:"trl.DPOTrainer.train.resume_from_checkpoint",description:`<strong>resume_from_checkpoint</strong> (<code>str</code> or <code>bool</code>, <em>optional</em>) &#x2014;
If a <code>str</code>, local path to a saved checkpoint as saved by a previous instance of <code>Trainer</code>. If a
<code>bool</code> and equals <code>True</code>, load the last checkpoint in <em>args.output_dir</em> as saved by a previous instance
of <code>Trainer</code>. If present, training will resume from the model/optimizer/scheduler states loaded here.`,name:"resume_from_checkpoint"},{anchor:"trl.DPOTrainer.train.trial",description:`<strong>trial</strong> (<code>optuna.Trial</code> or <code>dict[str, Any]</code>, <em>optional</em>) &#x2014;
The trial run or the hyperparameter dictionary for hyperparameter search.`,name:"trial"},{anchor:"trl.DPOTrainer.train.ignore_keys_for_eval",description:`<strong>ignore_keys_for_eval</strong> (<code>list[str]</code>, <em>optional</em>) &#x2014;
A list of keys in the output of your model (if it is a dictionary) that should be ignored when
gathering predictions for evaluation during the training.`,name:"ignore_keys_for_eval"}],source:"https://github.com/huggingface/trl/blob/vr_5320/transformers/trainer.py#L1322",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Object containing the global step count, training loss, and metrics.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>~trainer_utils.TrainOutput</code></p>
`}}),O=new is({props:{name:"save_model",anchor:"trl.DPOTrainer.save_model",parameters:[{name:"output_dir",val:": str | None = None"},{name:"_internal_call",val:": bool = False"}],source:"https://github.com/huggingface/trl/blob/vr_5320/transformers/trainer.py#L3739"}}),G=new is({props:{name:"push_to_hub",anchor:"trl.DPOTrainer.push_to_hub",parameters:[{name:"commit_message",val:": str | None = 'End of training'"},{name:"blocking",val:": bool = True"},{name:"token",val:": str | None = None"},{name:"revision",val:": str | None = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"trl.DPOTrainer.push_to_hub.commit_message",description:`<strong>commit_message</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;End of training&quot;</code>) &#x2014;
Message to commit while pushing.`,name:"commit_message"},{anchor:"trl.DPOTrainer.push_to_hub.blocking",description:`<strong>blocking</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether the function should return only when the <code>git push</code> has finished.`,name:"blocking"},{anchor:"trl.DPOTrainer.push_to_hub.token",description:`<strong>token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Token with write permission to overwrite Trainer&#x2019;s original args.`,name:"token"},{anchor:"trl.DPOTrainer.push_to_hub.revision",description:`<strong>revision</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The git revision to commit from. Defaults to the head of the &#x201C;main&#x201D; branch.`,name:"revision"},{anchor:"trl.DPOTrainer.push_to_hub.kwargs",description:`<strong>kwargs</strong> (<code>dict[str, Any]</code>, <em>optional</em>) &#x2014;
Additional keyword arguments passed along to <code>~Trainer.create_model_card</code>.`,name:"kwargs"}],source:"https://github.com/huggingface/trl/blob/vr_5320/transformers/trainer.py#L3986",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The URL of the repository where the model was pushed if <code>blocking=False</code>, or a <code>Future</code> object tracking the
progress of the commit if <code>blocking=True</code>.</p>
`}}),V=new Ws({props:{title:"BEMACallback",local:"trl.BEMACallback",headingTag:"h2"}}),R=new is({props:{name:"class trl.BEMACallback",anchor:"trl.BEMACallback",parameters:[{name:"update_freq",val:": int = 400"},{name:"ema_power",val:": float = 0.5"},{name:"bias_power",val:": float = 0.2"},{name:"lag",val:": int = 10"},{name:"update_after",val:": int = 0"},{name:"multiplier",val:": float = 1.0"},{name:"min_ema_multiplier",val:": float = 0.0"},{name:"device",val:": str = 'cpu'"},{name:"update_ref_model",val:": bool = False"},{name:"ref_model_update_freq",val:": int = 400"},{name:"ref_model_update_after",val:": int = 0"}],parametersDescription:[{anchor:"trl.BEMACallback.update_freq",description:`<strong>update_freq</strong> (<code>int</code>, <em>optional</em>, defaults to <code>400</code>) &#x2014;
Update the BEMA weights every X steps. Denoted this as {@html &quot;<span class="\\&quot;katex\\&quot;"><span class="\\&quot;katex-mathml\\&quot;"><math xmlns="\\&quot;http://www.w3.org/1998/Math/MathML\\&quot;"><semantics><mrow><mi>&#x3D5;</mi></mrow><annotation encoding="\\&quot;application/x-tex\\&quot;"> \\\\phi </annotation></semantics></math></span><span class="\\&quot;katex-html\\&quot;" aria-hidden="\\&quot;true\\&quot;"><span class="\\&quot;base\\&quot;"><span class="\\&quot;strut\\&quot;" style="\\&quot;height:0.8889em;vertical-align:-0.1944em;\\&quot;"></span><span class="\\&quot;mord" mathnormal\\">&#x3D5;</span></span></span></span>&quot;} in the paper.`,name:"update_freq"},{anchor:"trl.BEMACallback.ema_power",description:`<strong>ema_power</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.5</code>) &#x2014;
Power for the EMA decay factor. Denoted {@html &quot;<span class="\\&quot;katex\\&quot;"><span class="\\&quot;katex-mathml\\&quot;"><math xmlns="\\&quot;http://www.w3.org/1998/Math/MathML\\&quot;"><semantics><mrow><mi>&#x3BA;</mi></mrow><annotation encoding="\\&quot;application/x-tex\\&quot;"> \\\\kappa </annotation></semantics></math></span><span class="\\&quot;katex-html\\&quot;" aria-hidden="\\&quot;true\\&quot;"><span class="\\&quot;base\\&quot;"><span class="\\&quot;strut\\&quot;" style="\\&quot;height:0.4306em;\\&quot;"></span><span class="\\&quot;mord" mathnormal\\">&#x3BA;</span></span></span></span>&quot;} in the paper. To disable EMA, set this to <code>0.0</code>.`,name:"ema_power"},{anchor:"trl.BEMACallback.bias_power",description:`<strong>bias_power</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.2</code>) &#x2014;
Power for the BEMA scaling factor. Denoted {@html &quot;<span class="\\&quot;katex\\&quot;"><span class="\\&quot;katex-mathml\\&quot;"><math xmlns="\\&quot;http://www.w3.org/1998/Math/MathML\\&quot;"><semantics><mrow><mi>&#x3B7;</mi></mrow><annotation encoding="\\&quot;application/x-tex\\&quot;"> \\\\eta </annotation></semantics></math></span><span class="\\&quot;katex-html\\&quot;" aria-hidden="\\&quot;true\\&quot;"><span class="\\&quot;base\\&quot;"><span class="\\&quot;strut\\&quot;" style="\\&quot;height:0.625em;vertical-align:-0.1944em;\\&quot;"></span><span class="\\&quot;mord" mathnormal\\" style="\\&quot;margin-right:0.03588em;\\&quot;">&#x3B7;</span></span></span></span>&quot;} in the paper. To disable BEMA, set this to <code>0.0</code>.`,name:"bias_power"},{anchor:"trl.BEMACallback.lag",description:`<strong>lag</strong> (<code>int</code>, <em>optional</em>, defaults to <code>10</code>) &#x2014;
Initial offset in the weight decay schedule that controls early-stage smoothness by acting as a virtual
starting age for the updates. Denoted as {@html &quot;<span class="\\&quot;katex\\&quot;"><span class="\\&quot;katex-mathml\\&quot;"><math xmlns="\\&quot;http://www.w3.org/1998/Math/MathML\\&quot;"><semantics><mrow><mi>&#x3C1;</mi></mrow><annotation encoding="\\&quot;application/x-tex\\&quot;"> \\\\rho </annotation></semantics></math></span><span class="\\&quot;katex-html\\&quot;" aria-hidden="\\&quot;true\\&quot;"><span class="\\&quot;base\\&quot;"><span class="\\&quot;strut\\&quot;" style="\\&quot;height:0.625em;vertical-align:-0.1944em;\\&quot;"></span><span class="\\&quot;mord" mathnormal\\">&#x3C1;</span></span></span></span>&quot;} in the paper.`,name:"lag"},{anchor:"trl.BEMACallback.update_after",description:`<strong>update_after</strong> (<code>int</code>, <em>optional</em>, defaults to <code>0</code>) &#x2014;
Burn-in time before starting to update the BEMA weights. Denoted {@html &quot;<span class="\\&quot;katex\\&quot;"><span class="\\&quot;katex-mathml\\&quot;"><math xmlns="\\&quot;http://www.w3.org/1998/Math/MathML\\&quot;"><semantics><mrow><mi>&#x3C4;</mi></mrow><annotation encoding="\\&quot;application/x-tex\\&quot;"> \\\\tau </annotation></semantics></math></span><span class="\\&quot;katex-html\\&quot;" aria-hidden="\\&quot;true\\&quot;"><span class="\\&quot;base\\&quot;"><span class="\\&quot;strut\\&quot;" style="\\&quot;height:0.4306em;\\&quot;"></span><span class="\\&quot;mord" mathnormal\\" style="\\&quot;margin-right:0.1132em;\\&quot;">&#x3C4;</span></span></span></span>&quot;} in the paper.`,name:"update_after"},{anchor:"trl.BEMACallback.multiplier",description:`<strong>multiplier</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1.0</code>) &#x2014;
Initial value for the EMA decay factor. Denoted as {@html &quot;<span class="\\&quot;katex\\&quot;"><span class="\\&quot;katex-mathml\\&quot;"><math xmlns="\\&quot;http://www.w3.org/1998/Math/MathML\\&quot;"><semantics><mrow><mi>&#x3B3;</mi></mrow><annotation encoding="\\&quot;application/x-tex\\&quot;"> \\\\gamma </annotation></semantics></math></span><span class="\\&quot;katex-html\\&quot;" aria-hidden="\\&quot;true\\&quot;"><span class="\\&quot;base\\&quot;"><span class="\\&quot;strut\\&quot;" style="\\&quot;height:0.625em;vertical-align:-0.1944em;\\&quot;"></span><span class="\\&quot;mord" mathnormal\\" style="\\&quot;margin-right:0.05556em;\\&quot;">&#x3B3;</span></span></span></span>&quot;} in the paper.`,name:"multiplier"},{anchor:"trl.BEMACallback.min_ema_multiplier",description:`<strong>min_ema_multiplier</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.0</code>) &#x2014;
Minimum value for the EMA decay factor.`,name:"min_ema_multiplier"},{anchor:"trl.BEMACallback.device",description:`<strong>device</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;cpu&quot;</code>) &#x2014;
Device to use for the BEMA buffers, e.g. <code>&quot;cpu&quot;</code> or <code>&quot;cuda&quot;</code>. Note that in most cases, this device SHOULD
BE DIFFERENT from the device used for training in order to avoid OOM.`,name:"device"},{anchor:"trl.BEMACallback.update_ref_model",description:`<strong>update_ref_model</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to update the reference model with BEMA weights. This creates a lagged, smoothed version of the
main model as the reference model.`,name:"update_ref_model"},{anchor:"trl.BEMACallback.ref_model_update_freq",description:`<strong>ref_model_update_freq</strong> (<code>int</code>, <em>optional</em>, defaults to <code>400</code>) &#x2014;
Update the reference model with BEMA weights every this many steps.`,name:"ref_model_update_freq"},{anchor:"trl.BEMACallback.ref_model_update_after",description:`<strong>ref_model_update_after</strong> (<code>int</code>, <em>optional</em>, defaults to <code>0</code>) &#x2014;
Number of steps to wait before starting to update the reference model.`,name:"ref_model_update_after"}],source:"https://github.com/huggingface/trl/blob/vr_5320/trl/experimental/bema_for_ref_model/callback.py#L59"}}),L=new Ja({props:{anchor:"trl.BEMACallback.example",$$slots:{default:[Ua]},$$scope:{ctx:rs}}}),H=new Da({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/bema_for_reference_model.md"}}),{c(){h=p("meta"),F=n(),C=p("p"),x=n(),u(M.$$.fragment),i=n(),u($.$$.fragment),os=n(),N=p("p"),N.textContent=ma,cs=n(),u(W.$$.fragment),hs=n(),u(j.$$.fragment),gs=n(),u(I.$$.fragment),ds=n(),_=p("div"),u(Z.$$.fragment),js=n(),J=p("div"),u(Q.$$.fragment),Is=n(),S=p("p"),S.textContent=ia,Zs=n(),B=p("div"),u(O.$$.fragment),Qs=n(),K=p("p"),K.innerHTML=ra,Os=n(),ss=p("p"),ss.textContent=oa,Gs=n(),U=p("div"),u(G.$$.fragment),Vs=n(),as=p("p"),as.innerHTML=ca,us=n(),u(V.$$.fragment),fs=n(),r=p("div"),u(R.$$.fragment),Rs=n(),ts=p("p"),ts.innerHTML=ha,Hs=n(),es=p("p"),Xs=q(`BEMA computes model weights that scale like:
`),vs=new T(!1),Ys=n(),c=p("p"),Ss=q("where "),ys=new T(!1),bs=q(" is the current model weights, "),ws=new T(!1),_s=q(` is a snapshot of the model weights at the
first `),ns=p("code"),ns.textContent=ga,Ks=q(" step, "),xs=new T(!1),Ms=q(" is the exponential moving average of the model weights, and"),qs=new T(!1),ks=q(" is a scaling factor that decays with the number of steps "),$s=new T(!1),Es=q(` as
`),Ts=new T(!1),sa=n(),ls=p("p"),aa=q(`The EMA is computed as:
`),zs=new T(!1),ta=n(),D=p("p"),ea=q("where "),As=new T(!1),Cs=q(" is a decay factor that decays with the number of steps "),Bs=new T(!1),Ds=q(` as
`),Js=new T(!1),na=n(),u(L.$$.fragment),Us=n(),u(H.$$.fragment),Ls=n(),ms=p("p"),this.h()},l(s){const e=Ca("svelte-u9bgzb",document.head);h=m(e,"META",{name:!0,content:!0}),e.forEach(t),F=l(s),C=m(s,"P",{}),z(C).forEach(t),x=l(s),f(M.$$.fragment,s),i=l(s),f($.$$.fragment,s),os=l(s),N=m(s,"P",{"data-svelte-h":!0}),P(N)!=="svelte-1n2qlx2"&&(N.textContent=ma),cs=l(s),f(W.$$.fragment,s),hs=l(s),f(j.$$.fragment,s),gs=l(s),f(I.$$.fragment,s),ds=l(s),_=m(s,"DIV",{class:!0});var E=z(_);f(Z.$$.fragment,E),js=l(E),J=m(E,"DIV",{class:!0});var Fs=z(J);f(Q.$$.fragment,Fs),Is=l(Fs),S=m(Fs,"P",{"data-svelte-h":!0}),P(S)!=="svelte-1cilnet"&&(S.textContent=ia),Fs.forEach(t),Zs=l(E),B=m(E,"DIV",{class:!0});var ps=z(B);f(O.$$.fragment,ps),Qs=l(ps),K=m(ps,"P",{"data-svelte-h":!0}),P(K)!=="svelte-r8h4ov"&&(K.innerHTML=ra),Os=l(ps),ss=m(ps,"P",{"data-svelte-h":!0}),P(ss)!=="svelte-1e6bius"&&(ss.textContent=oa),ps.forEach(t),Gs=l(E),U=m(E,"DIV",{class:!0});var Ns=z(U);f(G.$$.fragment,Ns),Vs=l(Ns),as=m(Ns,"P",{"data-svelte-h":!0}),P(as)!=="svelte-8tudwd"&&(as.innerHTML=ca),Ns.forEach(t),E.forEach(t),us=l(s),f(V.$$.fragment,s),fs=l(s),r=m(s,"DIV",{class:!0});var d=z(r);f(R.$$.fragment,d),Rs=l(d),ts=m(d,"P",{"data-svelte-h":!0}),P(ts)!=="svelte-1rpbmy"&&(ts.innerHTML=ha),Hs=l(d),es=m(d,"P",{});var la=z(es);Xs=k(la,`BEMA computes model weights that scale like:
`),vs=A(la,!1),la.forEach(t),Ys=l(d),c=m(d,"P",{});var g=z(c);Ss=k(g,"where "),ys=A(g,!1),bs=k(g," is the current model weights, "),ws=A(g,!1),_s=k(g,` is a snapshot of the model weights at the
first `),ns=m(g,"CODE",{"data-svelte-h":!0}),P(ns)!=="svelte-1il68i0"&&(ns.textContent=ga),Ks=k(g," step, "),xs=A(g,!1),Ms=k(g," is the exponential moving average of the model weights, and"),qs=A(g,!1),ks=k(g," is a scaling factor that decays with the number of steps "),$s=A(g,!1),Es=k(g,` as
`),Ts=A(g,!1),g.forEach(t),sa=l(d),ls=m(d,"P",{});var pa=z(ls);aa=k(pa,`The EMA is computed as:
`),zs=A(pa,!1),pa.forEach(t),ta=l(d),D=m(d,"P",{});var X=z(D);ea=k(X,"where "),As=A(X,!1),Cs=k(X," is a decay factor that decays with the number of steps "),Bs=A(X,!1),Ds=k(X,` as
`),Js=A(X,!1),X.forEach(t),na=l(d),f(L.$$.fragment,d),d.forEach(t),Us=l(s),f(H.$$.fragment,s),Ls=l(s),ms=m(s,"P",{}),z(ms).forEach(t),this.h()},h(){Y(h,"name","hf:doc:metadata"),Y(h,"content",Pa),Y(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Y(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Y(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Y(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),vs.a=null,ys.a=bs,ws.a=_s,xs.a=Ms,qs.a=ks,$s.a=Es,Ts.a=null,zs.a=null,As.a=Cs,Bs.a=Ds,Js.a=null,Y(r,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(s,e){a(document.head,h),o(s,F,e),o(s,C,e),o(s,x,e),v(M,s,e),o(s,i,e),v($,s,e),o(s,os,e),o(s,N,e),o(s,cs,e),v(W,s,e),o(s,hs,e),v(j,s,e),o(s,gs,e),v(I,s,e),o(s,ds,e),o(s,_,e),v(Z,_,null),a(_,js),a(_,J),v(Q,J,null),a(J,Is),a(J,S),a(_,Zs),a(_,B),v(O,B,null),a(B,Qs),a(B,K),a(B,Os),a(B,ss),a(_,Gs),a(_,U),v(G,U,null),a(U,Vs),a(U,as),o(s,us,e),v(V,s,e),o(s,fs,e),o(s,r,e),v(R,r,null),a(r,Rs),a(r,ts),a(r,Hs),a(r,es),a(es,Xs),vs.m(ua,es),a(r,Ys),a(r,c),a(c,Ss),ys.m(fa,c),a(c,bs),ws.m(va,c),a(c,_s),a(c,ns),a(c,Ks),xs.m(ya,c),a(c,Ms),qs.m(ba,c),a(c,ks),$s.m(wa,c),a(c,Es),Ts.m(_a,c),a(r,sa),a(r,ls),a(ls,aa),zs.m(xa,ls),a(r,ta),a(r,D),a(D,ea),As.m(Ma,D),a(D,Cs),Bs.m(qa,D),a(D,Ds),Js.m(ka,D),a(r,na),v(L,r,null),o(s,Us,e),v(H,s,e),o(s,Ls,e),o(s,ms,e),Ps=!0},p(s,[e]){const E={};e&2&&(E.$$scope={dirty:e,ctx:s}),L.$set(E)},i(s){Ps||(y(M.$$.fragment,s),y($.$$.fragment,s),y(W.$$.fragment,s),y(j.$$.fragment,s),y(I.$$.fragment,s),y(Z.$$.fragment,s),y(Q.$$.fragment,s),y(O.$$.fragment,s),y(G.$$.fragment,s),y(V.$$.fragment,s),y(R.$$.fragment,s),y(L.$$.fragment,s),y(H.$$.fragment,s),Ps=!0)},o(s){b(M.$$.fragment,s),b($.$$.fragment,s),b(W.$$.fragment,s),b(j.$$.fragment,s),b(I.$$.fragment,s),b(Z.$$.fragment,s),b(Q.$$.fragment,s),b(O.$$.fragment,s),b(G.$$.fragment,s),b(V.$$.fragment,s),b(R.$$.fragment,s),b(L.$$.fragment,s),b(H.$$.fragment,s),Ps=!1},d(s){s&&(t(F),t(C),t(x),t(i),t(os),t(N),t(cs),t(hs),t(gs),t(ds),t(_),t(us),t(fs),t(r),t(Us),t(Ls),t(ms)),t(h),w(M,s),w($,s),w(W,s),w(j,s),w(I,s),w(Z),w(Q),w(O),w(G),w(V,s),w(R),w(L),w(H,s)}}}const Pa='{"title":"BEMA for Reference Model","local":"bema-for-reference-model","sections":[{"title":"Usage","local":"usage","sections":[],"depth":2},{"title":"DPOTrainer","local":"trl.DPOTrainer","sections":[],"depth":2},{"title":"BEMACallback","local":"trl.BEMACallback","sections":[],"depth":2}],"depth":1}';function Fa(rs){return Ea(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Oa extends za{constructor(h){super(),Aa(this,h,Fa,La,$a,{})}}export{Oa as component};