Buckets:
| import{s as jp,o as _p,n as Ip}from"../chunks/scheduler.37c15a92.js";import{S as Ap,i as kp,g as p,s as l,r,m as C,H as Da,A as Rp,h as i,f as e,c as n,j as w,u as c,x as m,n as x,E as Sa,k as bp,y,a as t,v as o,d as u,t as h,w as d}from"../chunks/index.2bf4358c.js";import{T as Gp}from"../chunks/Tip.363c041f.js";import{C as v}from"../chunks/CodeBlock.4e987730.js";import{H as g,E as Bp}from"../chunks/getInferenceSnippets.1837c472.js";function Qp(te){let M,z='Această secțiune se scufundă în detaliile tehnice și matematice ale GRPO. A fost scrisă de <a href="https://github.com/shirinyamani" rel="nofollow">Shirin Yamani</a>.';return{c(){M=p("p"),M.innerHTML=z},l(f){M=i(f,"P",{"data-svelte-h":!0}),m(M)!=="svelte-v63a7z"&&(M.innerHTML=z)},m(f,Wa){t(f,M,Wa)},p:Ip,d(f){f&&e(M)}}}function Pp(te){let M,z,f,Wa,j,le,$,ne,_,Nl="Să ne aprofundăm înțelegerea GRPO astfel încât să putem îmbunătăți procesul de antrenare al modelului nostru.",pe,I,Fl="GRPO evaluează direct răspunsurile generate de model comparându-le în cadrul grupurilor de generare pentru a optimiza modelul de politică, în loc să antreneze un model de valoare separat (Critic). Această abordare duce la o reducere semnificativă a costului computațional!",ie,A,Yl="GRPO poate fi aplicat la orice sarcină verificabilă unde corectitudinea răspunsului poate fi determinată. De exemplu, în raționamentul matematic, corectitudinea răspunsului poate fi ușor verificată prin compararea acestuia cu adevărul de bază.",me,k,Ol="Înainte de a ne scufunda în detaliile tehnice, să vizualizăm cum funcționează GRPO la un nivel înalt:",re,R,ql='<img src="./img/2.jpg" alt="deep"/>',ce,G,Kl="Acum că avem o prezentare vizuală, să descompunem cum funcționează GRPO pas cu pas.",oe,B,ue,Q,sn="Inovația principală a GRPO este abordarea sa de evaluare și învățare din multiple răspunsuri generate simultan. În loc să se bazeze pe un model de recompensă separat, compară rezultatele din același grup pentru a determina care ar trebui să fie întărite.",he,P,an="Să parcurgem fiecare pas al algoritmului în detaliu:",de,Z,ge,V,en="Primul pas este să genereze multiple răspunsuri posibile pentru fiecare întrebare. Aceasta creează un set divers de rezultate care pot fi comparate între ele.",ye,X,tn="Pentru fiecare întrebare $q$, modelul va genera $G$ rezultate (dimensiunea grupului) din politica antrenată: ${o<em>1, o_2, o_3, \\dots, o_G}\\pi</em>{\\theta_{\\text{old}}}$, $G=8$ unde fiecare $o_i$ reprezintă o completare din model.",Me,E,ve,L,ln="Pentru a face acest lucru concret, să ne uităm la o problemă aritmetică simplă:",fe,D,nn="<li><strong>Întrebarea</strong> $q$ : $\\text{Calculează}\\space2 + 2 \\times 6$</li> <li><strong>Rezultatele</strong> $(G = 8)$: ${o_1:14 \\text{ (corect)}, o_2:16 \\text{ (greșit)}, o_3:10 \\text{ (greșit)}, \\ldots, o_8:14 \\text{ (corect)}}$</li>",we,S,pn="Observă cum unele dintre răspunsurile generate sunt corecte (14) în timp ce altele sunt greșite (16 sau 10). Această diversitate este crucială pentru următorul pas.",be,W,$e,H,mn="Odată ce avem multiple răspunsuri, avem nevoie de o modalitate de a determina care sunt mai bune decât altele. Aici intervine calculul avantajului.",Je,N,Ue,F,rn="În primul rând, atribuim un scor de recompensă fiecărui răspuns generat. În acest exemplu, vom folosi un model de recompensă, dar după cum am învățat în secțiunea anterioară, putem folosi orice funcție care returnează recompense.",Te,Y,cn="Atribuie un scor RM fiecăruia dintre răspunsurile generate bazat pe corectitudine $r_i$ <em>(de exemplu, 1 pentru răspuns corect, 0 pentru răspuns greșit)</em> apoi pentru fiecare dintre $r_i$ calculează următoarea valoare de Avantaj",Ce,O,xe,q,Gl,ze,$p='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo>=</mo><mfrac><mrow><msub><mi>r</mi><mi>i</mi></msub><mo>−</mo><mtext>mean</mtext><mo stretchy="false">(</mo><mo stretchy="false">{</mo><msub><mi>r</mi><mn>1</mn></msub><mo separator="true">,</mo><msub><mi>r</mi><mn>2</mn></msub><mo separator="true">,</mo><mo>…</mo><mo separator="true">,</mo><msub><mi>r</mi><mi>G</mi></msub><mo stretchy="false">}</mo><mo stretchy="false">)</mo></mrow><mrow><mtext>std</mtext><mo stretchy="false">(</mo><mo stretchy="false">{</mo><msub><mi>r</mi><mn>1</mn></msub><mo separator="true">,</mo><msub><mi>r</mi><mn>2</mn></msub><mo separator="true">,</mo><mo>…</mo><mo separator="true">,</mo><msub><mi>r</mi><mi>G</mi></msub><mo stretchy="false">}</mo><mo stretchy="false">)</mo></mrow></mfrac></mrow><annotation encoding="application/x-tex">A_i = \\frac{r_i - \\text{mean}(\\{r_1, r_2, \\ldots, r_G\\})}{\\text{std}(\\{r_1, r_2, \\ldots, r_G\\})}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.363em;vertical-align:-0.936em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord text"><span class="mord">std</span></span><span class="mopen">({</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner">…</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">G</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">})</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord text"><span class="mord">mean</span></span><span class="mopen">({</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner">…</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">G</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">})</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.936em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>',je,K,_e,ss,on="Continuând cu exemplul nostru aritmetic pentru același exemplu de mai sus, imaginează-ți că avem 8 răspunsuri, 4 dintre care sunt corecte și restul greșite, prin urmare:",Ie,as,un="<li>Media Grupului: $mean(r_i) = 0.5$</li> <li>Std: $std(r_i) = 0.53$</li> <li>Valoarea Avantajului:<ul><li>Răspuns corect: $A_i = \\frac{1 - 0.5}{0.53}= 0.94$</li> <li>Răspuns greșit: $A_i = \\frac{0 - 0.5}{0.53}= -0.94$</li></ul></li>",Ae,es,ke,ts,hn="Acum că am calculat valorile avantajului, să înțelegem ce înseamnă:",Re,ls,dn="Această standardizare (adică ponderarea $A_i$) permite modelului să evalueze performanța relativă a fiecărui răspuns, ghidând procesul de optimizare să favorizeze răspunsurile care sunt mai bune decât media (recompensă mare) și să descurajeze pe cele care sunt mai rele. De exemplu, dacă $A_i > 0$, atunci $o_i$ este un răspuns mai bun decât nivelul mediu din grupul său; și dacă $A_i < 0$, atunci $o_i$ atunci calitatea răspunsului este mai mică decât media (adică calitate/performanță slabă).",Ge,ns,gn="Pentru exemplul de mai sus, dacă $A_i = 0.94 \\text{(rezultat corect)}$ atunci în timpul pașilor de optimizare probabilitatea sa de generare va fi crescută.",Be,ps,yn="Cu valorile avantajului noastre calculate, suntem acum gata să actualizăm politica.",Qe,is,Pe,ms,Mn="Pasul final este să folosim aceste valori ale avantajului pentru a actualiza modelul nostru astfel încât să devină mai probabil să genereze răspunsuri bune în viitor.",Ze,rs,Bl,Ve,Jp='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>J</mi><mrow><mi>G</mi><mi>R</mi><mi>P</mi><mi>O</mi></mrow></msub><mo stretchy="false">(</mo><mi>θ</mi><mo stretchy="false">)</mo><mo>=</mo><mrow><mo fence="true">[</mo><mfrac><mn>1</mn><mi>G</mi></mfrac><munderover><mo>∑</mo><mrow><mi>i</mi><mo>=</mo><mn>1</mn></mrow><mi>G</mi></munderover><mi>min</mi><mo></mo><mrow><mo fence="true">(</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><msub><mi>A</mi><mi>i</mi></msub><mo separator="true">,</mo><mtext>clip</mtext><mrow><mo fence="true">(</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><mo separator="true">,</mo><mn>1</mn><mo>−</mo><mi>ϵ</mi><mo separator="true">,</mo><mn>1</mn><mo>+</mo><mi>ϵ</mi><mo fence="true">)</mo></mrow><msub><mi>A</mi><mi>i</mi></msub><mo fence="true">)</mo></mrow><mo fence="true">]</mo></mrow><mo>−</mo><mi>β</mi><msub><mi>D</mi><mrow><mi>K</mi><mi>L</mi></mrow></msub><mo stretchy="false">(</mo><msub><mi>π</mi><mi>θ</mi></msub><mi mathvariant="normal">∣</mi><mi mathvariant="normal">∣</mi><msub><mi>π</mi><mrow><mi>r</mi><mi>e</mi><mi>f</mi></mrow></msub><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">J_{GRPO}(\\theta) = \\left[\\frac{1}{G} \\sum_{i=1}^{G} \\min \\left( \\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)} A_i, \\text{clip}\\left( \\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)}, 1 - \\epsilon, 1 + \\epsilon \\right) A_i \\right)\\right]- \\beta D_{KL}(\\pi_{\\theta} || \\pi_{ref})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.09618em;">J</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0962em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">GRPO</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.02778em;">θ</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:3.106em;vertical-align:-1.2777em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size4">[</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">G</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.8283em;"><span style="top:-1.8723em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">i</span><span class="mrel mtight">=</span><span class="mord mtight">1</span></span></span></span><span style="top:-3.05em;"><span class="pstrut" style="height:3.05em;"></span><span><span class="mop op-symbol large-op">∑</span></span></span><span style="top:-4.3em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">G</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:1.2777em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop">min</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0278em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.2559em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.9419em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord text"><span class="mord">clip</span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0278em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.2559em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.9419em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal">ϵ</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal">ϵ</span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size4">]</span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.0361em;vertical-align:-0.2861em;"></span><span class="mord mathnormal" style="margin-right:0.05278em;">β</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">D</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.07153em;">K</span><span class="mord mathnormal mtight">L</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣∣</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">re</span><span class="mord mathnormal mtight" style="margin-right:0.10764em;">f</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span><span class="mclose">)</span></span></span></span></span>',Xe,cs,vn="Această formulă ar putea părea intimidantă la început, dar este construită din mai multe componente care fiecare servesc un scop important. Să le descompunem una câte una.",Ee,os,Le,us,fn="Funcția de actualizare GRPO combină mai multe tehnici pentru a asigura o învățare stabilă și eficientă. Să examinăm fiecare componentă:",De,hs,Se,ds,wn="Raportul de probabilitate este definit ca:",We,gs,bn="$\\left(\\frac{\\pi<em>{\\theta}(o_i|q)}{\\pi</em>{\\theta_{old}}(o_i|q)}\\right)$",He,ys,$n="Intuitiv, formula compară cât de mult diferă probabilitatea de răspuns a modelului nou de probabilitatea de răspuns a modelului vechi în timp ce încorporează o preferință pentru răspunsuri care îmbunătățesc rezultatul așteptat.",Ne,Ms,Fe,vs,Jn="<li>Dacă $\\text{raport} > 1$, modelul nou atribuie o probabilitate mai mare răspunsului $o_i$ decât modelul vechi.</li> <li>Dacă $\\text{raport} < 1$, modelul nou atribuie o probabilitate mai mică lui $o_i$</li>",Ye,fs,Un="Acest raport ne permite să controlăm cât de mult se schimbă modelul la fiecare pas, ceea ce ne duce la următoarea componentă.",Oe,ws,qe,bs,Tn="Funcția de tăiere este definită ca:",Ke,$s,Cn="$\\text{clip}\\left( \\frac{\\pi<em>{\\theta}(o_i|q)}{\\pi</em>{\\theta_{old}}(o_i|q)}, 1 - \\epsilon, 1 + \\epsilon\\right)$",st,Js,xn="Limitează raportul discutat mai sus să fie în intervalul $[1 - \\epsilon, 1 + \\epsilon]$ pentru a evita/controla schimbări drastice sau actualizări nebunești și să nu pășească prea departe de politica veche. Cu alte cuvinte, limitează cât de mult poate crește raportul de probabilitate pentru a ajuta la menținerea stabilității prin evitarea actualizărilor care împing modelul nou prea departe de cel vechi.",at,Us,et,Ts,zn="Să ne uităm la două scenarii diferite pentru a înțelege mai bine această funcție de tăiere:",tt,Cs,jn="<li><strong>Cazul 1</strong>: dacă noua politică are o probabilitate de 0.9 pentru un răspuns specific și vechea politică are o probabilitate de 0.5, înseamnă că acest răspuns este întărit de noua politică să aibă o probabilitate mai mare, dar într-o limită controlată care este tăierea pentru a-și strânge mâinile să nu devină drastică <ul><li>$\\text{Raport}: \\frac{\\pi<em>{\\theta}(o_i|q)}{\\pi</em>{\\theta_{old}}(o_i|q)} = \\frac{0.9}{0.5} = 1.8 → \\text{Clip}\\space1.2$ (limita superioară 1.2)</li></ul></li> <li><strong>Cazul 2</strong>: Dacă noua politică nu este în favoarea unui răspuns (probabilitate mai mică de exemplu 0.2), însemnând dacă răspunsul nu este benefic creșterea ar putea fi incorectă, și modelul ar fi penalizat.<ul><li>$\\text{Raport}: \\frac{\\pi<em>{\\theta}(o_i|q)}{\\pi</em>{\\theta_{old}}(o_i|q)} = \\frac{0.2}{0.5} = 0.4 →\\text{Clip}\\space0.8$ (limita inferioară 0.8)</li></ul></li>",lt,xs,nt,zs,_n="<li>Formula încurajează modelul nou să favorizeze răspunsuri pe care modelul vechi le-a subponderat <strong>dacă ele îmbunătățesc rezultatul</strong>.</li> <li>Dacă modelul vechi deja favoriza un răspuns cu o probabilitate mare, modelul nou poate totuși să-l întărească <strong>dar doar într-o limită controlată $[1 - \\epsilon, 1 + \\epsilon]$, $\\text{(de exemplu, }\\epsilon = 0.2, \\space \\text{deci} \\space [0.8-1.2])$</strong>.</li> <li>Dacă modelul vechi a supraestimat un răspuns care performează prost, modelul nou este <strong>descurajat</strong> să mențină acea probabilitate mare.</li> <li>Prin urmare, intuitiv, prin încorporarea raportului de probabilitate, funcția obiectiv asigură că actualizările politicii sunt proporționale cu avantajul $A_i$ în timp ce sunt moderate pentru a preveni schimbări drastice.</li>",pt,js,In="În timp ce funcția de tăiere ajută la prevenirea schimbărilor drastice, avem nevoie de încă o măsură de protecție pentru a ne asigura că modelul nostru nu deviază prea departe de comportamentul său original.",it,_s,mt,Is,An="Termenul de divergență KL este:",rt,As,kn="$\\beta D<em>{KL}(\\pi</em>{\\theta} || \\pi_{ref})$",ct,ks,Rn="În termenul de divergență KL, $\\pi<em>{ref}$ este practic rezultatul modelului pre-actualizare, <code>per_token_logps</code> și $\\pi</em>{\\theta}$ este rezultatul modelului nou, <code>new_per_token_logps</code>. Teoretic, divergența KL este minimizată pentru a preveni modelul să devieze prea departe de comportamentul său original în timpul optimizării. Aceasta ajută să găsească un echilibru între îmbunătățirea performanței bazată pe semnalul de recompensă și menținerea coerenței. În acest context, minimizarea divergenței KL reduce riscul ca modelul să genereze text fără sens sau, în cazul raționamentului matematic, să producă răspunsuri extrem de incorecte.",ot,Rs,ut,Gs,Gn="<li>O penalitate de divergență KL păstrează rezultatele modelului aproape de distribuția sa originală, prevenind schimbări extreme.</li> <li>În loc să derive către rezultate complet iraționale, modelul și-ar rafina înțelegerea în timp ce încă permite unele explorări</li>",ht,Bs,dt,Qs,Bn="Pentru cei interesați de detaliile matematice, să ne uităm la definiția formală:",gt,J,Ql,yt,Up='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>D</mi><mrow><mi>K</mi><mi>L</mi></mrow></msub><mo stretchy="false">(</mo><mi>P</mi><mi mathvariant="normal">∣</mi><mi mathvariant="normal">∣</mi><mi>Q</mi><mo stretchy="false">)</mo><mo>=</mo><munder><mo>∑</mo><mrow><mi>x</mi><mo>∈</mo><mi>X</mi></mrow></munder><mi>P</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo><mi>log</mi><mo></mo><mfrac><mrow><mi>P</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo></mrow><mrow><mi>Q</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo></mrow></mfrac></mrow><annotation encoding="application/x-tex">D_{KL}(P || Q) = \\sum_{x \\in X} P(x) \\log \\frac{P(x)}{Q(x)}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">D</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.07153em;">K</span><span class="mord mathnormal mtight">L</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mord">∣∣</span><span class="mord mathnormal">Q</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.7487em;vertical-align:-1.3217em;"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.05em;"><span style="top:-1.8557em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">x</span><span class="mrel mtight">∈</span><span class="mord mathnormal mtight" style="margin-right:0.07847em;">X</span></span></span></span><span style="top:-3.05em;"><span class="pstrut" style="height:3.05em;"></span><span><span class="mop op-symbol large-op">∑</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:1.3217em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop">lo<span style="margin-right:0.01389em;">g</span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">Q</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.936em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>',Mt,vt,Ps,ft,Zs,Qn="Coeficientul $\\beta$ controlează cât de puternic impunem constrângerea divergenței KL:",wt,Vs,Pn='<li><strong>$\\beta$ Mai Mare (Penalitate KL Mai Puternică)</strong><ul><li>Mai multă constrângere asupra actualizărilor politicii. Modelul rămâne aproape de distribuția sa de referință.</li> <li>Poate încetini adaptarea: Modelul ar putea avea dificultăți să exploreze răspunsuri mai bune.</li></ul></li> <li><strong>$\\beta$ Mai Mic (Penalitate KL Mai Slabă)</strong><ul><li>Mai multă libertate să actualizeze politica: Modelul poate devia mai mult de la referință.</li> <li>Adaptare mai rapidă dar risc de instabilitate: Modelul ar putea învăța comportamente de hack-uire a recompenselor.</li> <li>Risc de supra-optimizare: Dacă modelul de recompensă este defectuos, politica ar putea genera rezultate fără sens.</li></ul></li> <li><strong>Originala</strong> <a href="https://arxiv.org/abs/2402.03300" rel="nofollow">DeepSeekMath</a> lucrare a setat acest $\\beta= 0.04$</li>',bt,Xs,Zn="Acum că înțelegem componentele GRPO, să vedem cum funcționează împreună într-un exemplu complet.",$t,Es,Jt,Ls,Vn="Pentru a ne solidifica înțelegerea GRPO, să parcurgem un exemplu complet de la început la sfârșit.",Ut,Ds,Tt,Ha,Ct,Tp='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mrow><mover accent="true"><mtext>I</mtext><mo>ˆ</mo></mover><mtext>: Calculeaz</mtext><mover accent="true"><mtext>a</mtext><mo>˘</mo></mover></mrow><mtext> </mtext><mn>2</mn><mo>+</mo><mn>2</mn><mo>×</mo><mn>6</mn></mrow><annotation encoding="application/x-tex">\\text{Î: Calculează}\\space2 + 2 \\times 6</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.0301em;vertical-align:-0.0833em;"></span><span class="mord text"><span class="mord accent"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.9468em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord">I</span></span><span style="top:-3.2523em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.25em;"><span class="mord">ˆ</span></span></span></span></span></span></span><span class="mord">: Calculeaz</span><span class="mord accent"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.6944em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord">a</span></span><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.25em;"><span class="mord">˘</span></span></span></span></span></span></span></span><span class="mspace"> </span><span class="mord">2</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.7278em;vertical-align:-0.0833em;"></span><span class="mord">2</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">6</span></span></span></span></span>',xt,Ss,zt,Ws,Xn="În primul rând, generăm multiple răspunsuri din modelul nostru:",jt,Hs,Pl,_t,Cp='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>o</mi><mn>1</mn></msub><mo>:</mo><mn>14</mn><mo stretchy="false">(</mo><mi>c</mi><mi>o</mi><mi>r</mi><mi>e</mi><mi>c</mi><mi>t</mi><mo stretchy="false">)</mo><mo separator="true">,</mo><msub><mi>o</mi><mn>2</mn></msub><mo>:</mo><mn>10</mn><mo stretchy="false">(</mo><mi>g</mi><mi>r</mi><mi>e</mi><mtext>ș</mtext><mi>i</mi><mi>t</mi><mo stretchy="false">)</mo><mo separator="true">,</mo><msub><mi>o</mi><mn>3</mn></msub><mo>:</mo><mn>16</mn><mo stretchy="false">(</mo><mi>g</mi><mi>r</mi><mi>e</mi><mtext>ș</mtext><mi>i</mi><mi>t</mi><mo stretchy="false">)</mo><mo separator="true">,</mo><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><msub><mi>o</mi><mi>G</mi></msub><mo>:</mo><mn>14</mn><mo stretchy="false">(</mo><mi>c</mi><mi>o</mi><mi>r</mi><mi>e</mi><mi>c</mi><mi>t</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">{o_1:14(corect), o_2:10 (greșit), o_3:16 (greșit), ... o_G:14(corect)}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">14</span><span class="mopen">(</span><span class="mord mathnormal">corec</span><span class="mord mathnormal">t</span><span class="mclose">)</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">10</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord mathnormal">re</span><span class="mord latin_fallback">ș</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mclose">)</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">3</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">16</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord mathnormal">re</span><span class="mord latin_fallback">ș</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mclose">)</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">...</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">G</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">14</span><span class="mopen">(</span><span class="mord mathnormal">corec</span><span class="mord mathnormal">t</span><span class="mclose">)</span></span></span></span></span></span>',It,Ns,At,Fs,En="Apoi, calculăm valorile avantajului pentru a determina care răspunsuri sunt mai bune decât media:",kt,b,Na,Zl,Rt,xp='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>m</mi><mi>e</mi><mi>a</mi><mi>n</mi><mo stretchy="false">(</mo><msub><mi>r</mi><mi>i</mi></msub><mo stretchy="false">)</mo><mo>=</mo><mn>0.5</mn></mrow><annotation encoding="application/x-tex">mean(r_i) = 0.5</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">m</span><span class="mord mathnormal">e</span><span class="mord mathnormal">an</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.5</span></span></span></span></span>',Vl,Fa,Ln="Std: $$std(r_i) = 0.53$$",Xl,Ya,Dn="Valoarea Avantajului:<ul><li>Răspuns corect: $A_i = \\frac{1 - 0.5}{0.53}= 0.94$</li> <li>Răspuns greșit: $A_i = \\frac{0 - 0.5}{0.53}= -0.94$</li></ul>",Gt,Ys,Bt,Os,Sn="În final, actualizăm modelul nostru pentru a întări răspunsurile corecte:",Qt,U,T,El,Oa,Wn="{\\theta",Ll,Pt,zp='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mtext>Raport</mtext><mo>:</mo><mfrac><mn>0.7</mn><mn>0.5</mn></mfrac><mo>=</mo><mn>1.4</mn><mo>→</mo><mrow><mtext>dup</mtext><mover accent="true"><mtext>a</mtext><mo>˘</mo></mover><mtext> Clip</mtext></mrow><mtext> </mtext><mn>1.2</mn><mtext> </mtext><mo stretchy="false">(</mo><mi>ϵ</mi><mo>=</mo><mn>0.2</mn><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">\\text{Raport}: \\frac{0.7}{0.5} = 1.4 →\\text{după Clip}\\space1.2 \\space (\\epsilon = 0.2)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8778em;vertical-align:-0.1944em;"></span><span class="mord text"><span class="mord">Raport</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">0.5</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">0.7</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">1.4</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">→</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord text"><span class="mord">dup</span><span class="mord accent"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.6944em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord">a</span></span><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.25em;"><span class="mord">˘</span></span></span></span></span></span></span><span class="mord"> Clip</span></span><span class="mspace"> </span><span class="mord">1.2</span><span class="mspace"> </span><span class="mopen">(</span><span class="mord mathnormal">ϵ</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">0.2</span><span class="mclose">)</span></span></span></span></span>',Dl,qa,Hn="Apoi când funcția țintă este re-ponderată, modelul tinde să întărească generarea rezultatului corect, și $\\text{Divergența KL}$ limitează deviația de la politica de referință.",Zt,qs,Nn="Cu înțelegerea teoretică la locul ei, să vedem cum poate fi implementat GRPO în cod.",Vt,Ks,Xt,sa,Fn="Să punem totul împreună într-un exemplu practic. Următorul cod demonstrează cum să implementezi GRPO în PyTorch.",Et,aa,Lt,ea,Yn="În primul rând, trebuie să încărcăm un model și să generăm multiple răspunsuri pentru o întrebare dată:",Dt,ta,St,la,On="această Generare inițială (Înainte de Orice Pași) va produce ceva de genul:",Wt,na,Ht,pa,Nt,ia,qn="Acum, trebuie să determinăm care răspunsuri sunt corecte și să atribuim recompense în consecință:",Ft,ma,Kn="Cu GRPO, cu același prompt de eșantion, generăm multiple completări. Deci, de exemplu, pentru prompt-urile noastre de <code>"Rezolvă y = 2x + 1 pentru x = 2, y = "</code> și <code>Rezolvă y = 2x + 1 pentru x = 4, y = "</code> avem două grupuri de rezultate generate pentru prompt-ul dat unul este să zicem",Yt,ra,sp="<li><code>[5, 6, 7, 5]</code> și celălalt este</li> <li><code>[10, 2, 9, 9]</code> în timp ce răspunsul corect este 5 și 9.</li>",Ot,ca,ap="Observă că în practică aceste scoruri de recompensă sunt obținute printr-o funcție de recompensă bazată pe reguli care atribuie recompense bazate pe corectitudinea răspunsului sau un model mai complex bazat pe rețele neuronale care poate fi antrenat să atribuie recompense bazate pe corectitudinea răspunsului sau un amestec din ambele. Dar pentru simplitate să spunem că recompensa noastră per răspuns este 1 dacă răspunsul este corect și 0 dacă este greșit, prin urmare:",qt,oa,Kt,ua,ep="apoi obținem media și std-ul grupului de recompense;",sl,ha,al,da,tp="aceasta va produce:",el,ga,tl,ya,lp="Acum putem calcula valorile avantajului pentru fiecare răspuns:",ll,Ma,nl,va,np="aceasta va produce:",pl,fa,il,wa,pp="care vine din formula Avantajului de mai sus, deci:",ml,ba,rl,$a,ip="cu toate acestea, forma aici este <code>(B*G,) = (8,)</code> dar în practică, avem nevoie să avem forma de <code>(B, G) = (2, 4)</code> pentru a se potrivi cu forma logits, nu? Prin urmare, trebuie să unsqueeze tensor-ul avantajelor pentru a avea forma de <code>(B*G, 1) = (8, 1)</code> pentru a se potrivi cu forma logits.",cl,Ja,ol,Ua,mp="care va produce:",ul,Ta,hl,Ca,rp="acum suntem bine, să trecem la următorul pas de actualizare a modelului de politică bazat pe valorile avantajului.",dl,xa,gl,za,cp="În final, folosim valorile avantajului pentru a actualiza modelul nostru:",yl,ja,Ml,_a,op="Observă că <code>per_token_logps</code> poate fi obținut prin trecerea rezultatelor generate la model și obținerea logits-urilor și apoi aplicarea funcției softmax pentru a obține probabilitățile <code>F.softmax(logits, dim=-1)</code>.",vl,Ia,fl,Aa,up="<code>per_token_kl</code> poate fi de asemenea calculat după cum urmează:",wl,ka,bl,Ra,hp='Exemplul complet poate fi găsit <a href="./basic_example.py">aici</a>. GRPO este de asemenea implementat de excelenta echipă TRL, poți verifica implementarea <a href="https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_trainer.py" rel="nofollow">TRL/GRPO_trainer</a> pentru mai multe detalii.',$l,Ga,Jl,Ba,dp="Felicitări! Acum ai învățat despre Optimizarea Relativă a Politicii de Grup (GRPO). Pentru a recapitula ce am acoperit:",Ul,Qa,gp="<li>GRPO compară multiple rezultate în cadrul unui grup pentru a determina care sunt mai bune decât altele, fără a necesita un model de valoare separat.</li> <li>Calculul avantajului standardizează recompensele pentru a identifica care răspunsuri sunt peste sau sub medie.</li> <li>Actualizarea politicii folosește o funcție obiectiv tăiată cu o penalitate de divergență KL pentru a asigura învățarea stabilă.</li>",Tl,Pa,yp="Această abordare este deosebit de puternică pentru sarcinile de raționament matematic, unde corectitudinea poate fi verificată obiectiv. Metoda GRPO permite antrenare mai eficientă comparativ cu abordările RLHF tradiționale care necesită un model critic separat.",Cl,Za,Mp="Pe măsură ce continui să explorezi GRPO, consideră să experimentezi cu diferite dimensiuni de grup, funcții de recompensă și coeficienți de penalitate KL pentru a vedea cum afectează performanța modelului tău.",xl,Va,vp="Antrenare fericită! 🚀",zl,Xa,jl,Ea,fp='<li><a href="https://github.com/natolambert/rlhf-book" rel="nofollow">Cartea RLHF de Nathan Lambert</a></li> <li><a href="https://huggingface.co/papers/2412.19437" rel="nofollow">Raportul Tehnic DeepSeek-V3</a></li> <li><a href="https://huggingface.co/papers/2402.03300" rel="nofollow">DeepSeekMath</a></li>',_l,La,Il,ae,Al;return j=new g({props:{title:"Înțelegerea Avansată a Optimizării Relative a Politicii de Grup (GRPO) în DeepSeekMath",local:"înțelegerea-avansată-a-optimizării-relative-a-politicii-de-grup-grpo-în-deepseekmath",headingTag:"h1"}}),$=new Gp({props:{$$slots:{default:[Qp]},$$scope:{ctx:te}}}),B=new g({props:{title:"Algoritmul GRPO",local:"algoritmul-grpo",headingTag:"h2"}}),Z=new g({props:{title:"Pasul 1: Eșantionarea Grupului",local:"pasul-1-eșantionarea-grupului",headingTag:"h3"}}),E=new g({props:{title:"Exemplu:",local:"exemplu",headingTag:"h4"}}),W=new g({props:{title:"Pasul 2: Calculul Avantajului",local:"pasul-2-calculul-avantajului",headingTag:"h3"}}),N=new g({props:{title:"Distribuția Recompenselor:",local:"distribuția-recompenselor",headingTag:"h4"}}),O=new g({props:{title:"Formula Valorii Avantajului:",local:"formula-valorii-avantajului",headingTag:"h4"}}),K=new g({props:{title:"Exemplu:",local:"exemplu",headingTag:"h4"}}),es=new g({props:{title:"Interpretarea:",local:"interpretarea",headingTag:"h4"}}),is=new g({props:{title:"Pasul 3: Actualizarea Politicii",local:"pasul-3-actualizarea-politicii",headingTag:"h3"}}),os=new g({props:{title:"Componentele Cheie ale Funcției Țintă",local:"componentele-cheie-ale-funcției-țintă",headingTag:"h2"}}),hs=new g({props:{title:"1. Raportul de Probabilitate",local:"1-raportul-de-probabilitate",headingTag:"h3"}}),Ms=new g({props:{title:"Interpretarea:",local:"interpretarea",headingTag:"h4"}}),ws=new g({props:{title:"2. Funcția de Tăiere",local:"2-funcția-de-tăiere",headingTag:"h3"}}),Us=new g({props:{title:"Exemplu $\\space \\text{să presupunem}(\\epsilon = 0.2)$",local:"exemplu-space-textsă-presupunemepsilon--02",headingTag:"h4"}}),xs=new g({props:{title:"Interpretarea:",local:"interpretarea",headingTag:"h4"}}),_s=new g({props:{title:"3. Divergența KL",local:"3-divergența-kl",headingTag:"h3"}}),Rs=new g({props:{title:"Interpretarea",local:"interpretarea",headingTag:"h4"}}),Bs=new g({props:{title:"Definiția Matematică",local:"definiția-matematică",headingTag:"h4"}}),Ps=new g({props:{title:"Rolul Parametrului $\\beta$",local:"rolul-parametrului-beta",headingTag:"h4"}}),Es=new g({props:{title:"Exemplu Lucrat cu GRPO",local:"exemplu-lucrat-cu-grpo",headingTag:"h2"}}),Ds=new g({props:{title:"Problema Exemplu",local:"problema-exemplu",headingTag:"h3"}}),Ss=new g({props:{title:"Pasul 1: Eșantionarea Grupului",local:"pasul-1-eșantionarea-grupului",headingTag:"h3"}}),Ns=new g({props:{title:"Pasul 2: Calculul Avantajului",local:"pasul-2-calculul-avantajului",headingTag:"h3"}}),Ys=new g({props:{title:"Pasul 3: Actualizarea Politicii",local:"pasul-3-actualizarea-politicii",headingTag:"h3"}}),Ks=new g({props:{title:"Exemplu de Implementare",local:"exemplu-de-implementare",headingTag:"h2"}}),aa=new g({props:{title:"1. Încărcarea Modelului și Generarea Răspunsurilor",local:"1-încărcarea-modelului-și-generarea-răspunsurilor",headingTag:"h3"}}),ta=new v({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaC5ubi5mdW5jdGlvbmFsJTIwYXMlMjBGJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQSUyMyUyMCVDMyU4RW5jYXJjJUM0JTgzJTIwbW9kZWx1bCUyMCVDOCU5OWklMjB0b2tlbml6ZXItdWwlMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyUXdlbiUyRlF3ZW4yLU1hdGgtMS41QiUyMiUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSklMEFtb2RlbC5ldmFsKCklMEElMEElMjMlMjBNdXQlQzQlODMlMjBtb2RlbHVsJTIwcGUlMjBHUFUlMjBkYWMlQzQlODMlMjBlc3RlJTIwZGlzcG9uaWJpbCUwQWRldmljZSUyMCUzRCUyMHRvcmNoLmRldmljZSglMjJjdWRhJTIyJTIwaWYlMjB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpJTIwZWxzZSUyMCUyMmNwdSUyMiklMEFtb2RlbC50byhkZXZpY2UpJTBBJTBBJTIzJTIwUHJvbXB0JTIwZGUlMjBpbnRyYXJlJTBBcHJvbXB0JTIwJTNEJTIwJTIyUmV6b2x2JUM0JTgzJTIweSUyMCUzRCUyMDJ4JTIwJTJCJTIwMSUyMHBlbnRydSUyMHglMjAlM0QlMjAyJTJDJTIweSUyMCUzRCUyMCUyMiUyMCUyMCUyMyUyMFIlQzQlODNzcHVucyUyMGNvcmVjdCUzQSUyMDUlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUyQyUyMHBhZGRpbmclM0RUcnVlKSUwQWlucHV0X2lkcyUyMCUzRCUyMGlucHV0cyU1QiUyMmlucHV0X2lkcyUyMiU1RC50byhkZXZpY2UpJTIwJTIwJTIzJTIwRm9ybWElM0ElMjAoMSUyQyUyMHByb21wdF9sZW4pJTBBYXR0ZW50aW9uX21hc2slMjAlM0QlMjBpbnB1dHMlNUIlMjJhdHRlbnRpb25fbWFzayUyMiU1RC50byhkZXZpY2UpJTBBJTBBJTIzJTIwUGFzdWwlMjAxJTNBJTIwR2VuZXJlYXolQzQlODMlMjA4JTIwciVDNCU4M3NwdW5zdXJpJTIwKEIlMjAlM0QlMjAyJTIwZ3J1cHVyaSUyQyUyMEclMjAlM0QlMjA0JTIwciVDNCU4M3NwdW5zdXJpJTIwcGVyJTIwZ3J1cCklMEFiYXRjaF9zaXplJTJDJTIwbnVtX2dlbmVyYXRpb25zJTIwJTNEJTIwMiUyQyUyMDQlMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoJTBBJTIwJTIwJTIwJTIwaW5wdXRfaWRzJTNEaW5wdXRfaWRzJTJDJTIwJTIwJTIzJTIwRm9ybWElM0ElMjAoMSUyQyUyMHByb21wdF9sZW4pJTBBJTIwJTIwJTIwJTIwYXR0ZW50aW9uX21hc2slM0RhdHRlbnRpb25fbWFzayUyQyUwQSUyMCUyMCUyMCUyMG1heF9uZXdfdG9rZW5zJTNEMSUyQyUyMCUyMCUyMyUyMHNlcV9sZW4lMjAlM0QlMjAxJTIwKHVuJTIwc2luZ3VyJTIwdG9rZW4lMjBwZXIlMjByJUM0JTgzc3B1bnMpJTBBJTIwJTIwJTIwJTIwbnVtX3JldHVybl9zZXF1ZW5jZXMlM0RiYXRjaF9zaXplJTIwKiUyMG51bV9nZW5lcmF0aW9ucyUyQyUyMCUyMCUyMyUyMDglMjByJUM0JTgzc3B1bnN1cmklMjAlQzMlQUVuJTIwdG90YWwlMEElMjAlMjAlMjAlMjBkb19zYW1wbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwdG9wX2slM0QxMCUyQyUwQSUyMCUyMCUyMCUyMHRlbXBlcmF0dXJlJTNEMC43JTJDJTBBJTIwJTIwJTIwJTIwcGFkX3Rva2VuX2lkJTNEdG9rZW5pemVyLmVvc190b2tlbl9pZCUyQyUwQSUyMCUyMCUyMCUyMHJldHVybl9kaWN0X2luX2dlbmVyYXRlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMG91dHB1dF9zY29yZXMlM0RUcnVlJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">import</span> torch.nn.functional <span class="hljs-keyword">as</span> F | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer | |
| <span class="hljs-comment"># Încarcă modelul și tokenizer-ul</span> | |
| model_name = <span class="hljs-string">"Qwen/Qwen2-Math-1.5B"</span> | |
| model = AutoModelForCausalLM.from_pretrained(model_name) | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| model.<span class="hljs-built_in">eval</span>() | |
| <span class="hljs-comment"># Mută modelul pe GPU dacă este disponibil</span> | |
| device = torch.device(<span class="hljs-string">"cuda"</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">"cpu"</span>) | |
| model.to(device) | |
| <span class="hljs-comment"># Prompt de intrare</span> | |
| prompt = <span class="hljs-string">"Rezolvă y = 2x + 1 pentru x = 2, y = "</span> <span class="hljs-comment"># Răspuns corect: 5</span> | |
| inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">"pt"</span>, padding=<span class="hljs-literal">True</span>) | |
| input_ids = inputs[<span class="hljs-string">"input_ids"</span>].to(device) <span class="hljs-comment"># Forma: (1, prompt_len)</span> | |
| attention_mask = inputs[<span class="hljs-string">"attention_mask"</span>].to(device) | |
| <span class="hljs-comment"># Pasul 1: Generează 8 răspunsuri (B = 2 grupuri, G = 4 răspunsuri per grup)</span> | |
| batch_size, num_generations = <span class="hljs-number">2</span>, <span class="hljs-number">4</span> | |
| outputs = model.generate( | |
| input_ids=input_ids, <span class="hljs-comment"># Forma: (1, prompt_len)</span> | |
| attention_mask=attention_mask, | |
| max_new_tokens=<span class="hljs-number">1</span>, <span class="hljs-comment"># seq_len = 1 (un singur token per răspuns)</span> | |
| num_return_sequences=batch_size * num_generations, <span class="hljs-comment"># 8 răspunsuri în total</span> | |
| do_sample=<span class="hljs-literal">True</span>, | |
| top_k=<span class="hljs-number">10</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| pad_token_id=tokenizer.eos_token_id, | |
| return_dict_in_generate=<span class="hljs-literal">True</span>, | |
| output_scores=<span class="hljs-literal">True</span>, | |
| )`,wrap:!1}}),na=new v({props:{code:"UmV6dWx0YXR1bCUyMDElM0ElMjA1LjAlMEFSZXp1bHRhdHVsJTIwMiUzQSUyMDYuMCUwQVJlenVsdGF0dWwlMjAzJTNBJTIwNy4wJTBBUmV6dWx0YXR1bCUyMDQlM0ElMjA1LjAlMEFSZXp1bHRhdHVsJTIwNSUzQSUyMDEwLjAlMEFSZXp1bHRhdHVsJTIwNiUzQSUyMDIuMCUwQVJlenVsdGF0dWwlMjA3JTNBJTIwNS4wJTBBUmV6dWx0YXR1bCUyMDglM0ElMjA1LjA=",highlighted:`Rezultatul 1: 5.0 | |
| Rezultatul 2: 6.0 | |
| Rezultatul 3: 7.0 | |
| Rezultatul 4: 5.0 | |
| Rezultatul 5: 10.0 | |
| Rezultatul 6: 2.0 | |
| Rezultatul 7: 5.0 | |
| Rezultatul 8: 5.0`,wrap:!1}}),pa=new g({props:{title:"2. Calcularea Recompenselor",local:"2-calcularea-recompenselor",headingTag:"h3"}}),oa=new v({props:{code:"cmV3YXJkXzElMjAlM0QlMjAlNUIxJTJDJTIwMCUyQyUyMDAlMkMlMjAxJTVEJTBBcmV3YXJkXzIlMjAlM0QlMjAlNUIwJTJDJTIwMCUyQyUyMDElMkMlMjAxJTVE",highlighted:`reward_1 = [<span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>] | |
| reward_2 = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]`,wrap:!1}}),ha=new v({props:{code:"JTIzJTIwRm9ybWElM0ElMjAoQiUyMColMjBHJTJDKSUyMCUzRCUyMCg4JTJDKSUyMHBlbnRydSUyMGMlQzQlODMlMjBhdmVtJTIwMiUyMGdydXB1cmklMjBkZSUyMDQlMjBnZW5lciVDNCU4M3JpJTIwcGUlMjBjYXJlJTIwbGUlMjBhcGxhdGl6JUM0JTgzbSUwQXJld2FyZHMlMjAlM0QlMjB0b3JjaC50ZW5zb3IoJTVCMSUyQyUyMDAlMkMlMjAwJTJDJTIwMSUyQyUyMDAlMkMlMjAwJTJDJTIwMSUyQyUyMDElNUQlMkMlMjBkdHlwZSUzRHRvcmNoLmZsb2F0MzIpJTBBbnVtX2dlbmVyYXRpb25zJTIwJTNEJTIwNCUwQSUwQSUyMyUyMFJlY29tcGVuc2UlMjBncnVwYXRlJTNBJTIwRm9ybWElMjAoQiUyQyUyMEcpJTIwJTNEJTIwMiUyQyUyMDQpJTBBcmV3YXJkc19ncm91cGVkJTIwJTNEJTIwcmV3YXJkcy52aWV3KC0xJTJDJTIwbnVtX2dlbmVyYXRpb25zKSUwQSUwQSUyMyUyME1lZGlhJTIwcGVyJTIwZ3J1cCUzQSUyMEZvcm1hJTIwKEIlMkMpJTIwJTNEJTIwKDIlMkMpJTBBbWVhbl9ncm91cGVkX3Jld2FyZHMlMjAlM0QlMjByZXdhcmRzX2dyb3VwZWQubWVhbihkaW0lM0QxKSUwQSUwQSUyMyUyMFN0ZCUyMHBlciUyMGdydXAlM0ElMjBGb3JtYSUyMChCJTJDKSUyMCUzRCUyMCgyJTJDKSUwQXN0ZF9ncm91cGVkX3Jld2FyZHMlMjAlM0QlMjByZXdhcmRzX2dyb3VwZWQuc3RkKGRpbSUzRDEpJTBBJTBBJTIzJTIwRGlmdXplYXolQzQlODMlMjBwZW50cnUlMjBhJTIwc2UlMjBwb3RyaXZpJTIwY3UlMjByZWNvbXBlbnNlbGUlMjAlQzglOTlpJTIwbm9ybWFsaXplYXolQzQlODMlM0ElMjBGb3JtYSUyMChCJTIwKiUyMEclMkMpJTIwJTNEJTIwKDglMkMpJTBBJTIzJTIwZGUlMjBjZSUyMGF2ZW0lMjBuZXZvaWUlMjBzJUM0JTgzJTIwZGlmdXolQzQlODNtJTNGJTIwcGVudHJ1JTIwYyVDNCU4MyUyMHRyZWJ1aWUlMjBzJUM0JTgzJTIwY2FsY3VsJUM0JTgzbSUyMHZhbG9yaWxlJTIwYXZhbnRhanVsdWklMjBwZW50cnUlMjBmaWVjYXJlJTIwciVDNCU4M3NwdW5zJTIwZGluJTIwZ3J1cCUwQW1lYW5fZ3JvdXBlZF9yZXdhcmRzJTIwJTNEJTIwbWVhbl9ncm91cGVkX3Jld2FyZHMucmVwZWF0X2ludGVybGVhdmUobnVtX2dlbmVyYXRpb25zJTJDJTIwZGltJTNEMCklMEFzdGRfZ3JvdXBlZF9yZXdhcmRzJTIwJTNEJTIwc3RkX2dyb3VwZWRfcmV3YXJkcy5yZXBlYXRfaW50ZXJsZWF2ZShudW1fZ2VuZXJhdGlvbnMlMkMlMjBkaW0lM0QwKQ==",highlighted:`<span class="hljs-comment"># Forma: (B * G,) = (8,) pentru că avem 2 grupuri de 4 generări pe care le aplatizăm</span> | |
| rewards = torch.tensor([<span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], dtype=torch.float32) | |
| num_generations = <span class="hljs-number">4</span> | |
| <span class="hljs-comment"># Recompense grupate: Forma (B, G) = 2, 4)</span> | |
| rewards_grouped = rewards.view(-<span class="hljs-number">1</span>, num_generations) | |
| <span class="hljs-comment"># Media per grup: Forma (B,) = (2,)</span> | |
| mean_grouped_rewards = rewards_grouped.mean(dim=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># Std per grup: Forma (B,) = (2,)</span> | |
| std_grouped_rewards = rewards_grouped.std(dim=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># Difuzează pentru a se potrivi cu recompensele și normalizează: Forma (B * G,) = (8,)</span> | |
| <span class="hljs-comment"># de ce avem nevoie să difuzăm? pentru că trebuie să calculăm valorile avantajului pentru fiecare răspuns din grup</span> | |
| mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(num_generations, dim=<span class="hljs-number">0</span>) | |
| std_grouped_rewards = std_grouped_rewards.repeat_interleave(num_generations, dim=<span class="hljs-number">0</span>)`,wrap:!1}}),ga=new v({props:{code:"UmVjb21wZW5zZSUyMEdydXBhdGUlM0ElMjB0ZW5zb3IoJTVCJTVCMS4lMkMlMjAwLiUyQyUyMDAuJTJDJTIwMS4lNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUIwLiUyQyUyMDAuJTJDJTIwMS4lMkMlMjAxLiU1RCU1RCklMEFNZWRpYSUyMHBlciUyMGdydXAlM0ElMjB0ZW5zb3IoJTVCMC41MDAwJTJDJTIwMC41MDAwJTVEKSUwQVN0ZCUyMHBlciUyMGdydXAlM0ElMjB0ZW5zb3IoJTVCMC41Nzc0JTJDJTIwMC41Nzc0JTVEKSUwQU1lZGlhJTIwRGlmdXphdCVDNCU4MyUzQSUyMHRlbnNvciglNUIwLjUwMDAlMkMlMjAwLjUwMDAlMkMlMjAwLjUwMDAlMkMlMjAwLjUwMDAlMkMlMjAwLjUwMDAlMkMlMjAwLjUwMDAlMkMlMjAwLjUwMDAlMkMlMjAwLjUwMDAlNUQpJTBBU3RkJTIwRGlmdXphdCUzQSUyMHRlbnNvciglNUIwLjU3NzQlMkMlMjAwLjU3NzQlMkMlMjAwLjU3NzQlMkMlMjAwLjU3NzQlMkMlMjAwLjU3NzQlMkMlMjAwLjU3NzQlMkMlMjAwLjU3NzQlMkMlMjAwLjU3NzQlNUQp",highlighted:`Recompense Grupate: tensor([[1., 0., 0., 1.], | |
| [0., 0., 1., 1.]]) | |
| Media per grup: tensor([0.5000, 0.5000]) | |
| Std per grup: tensor([0.5774, 0.5774]) | |
| Media Difuzată: tensor([0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000]) | |
| Std Difuzat: tensor([0.5774, 0.5774, 0.5774, 0.5774, 0.5774, 0.5774, 0.5774, 0.5774])`,wrap:!1}}),Ma=new v({props:{code:"JTIzJTIwQXZhbnRhamUlM0ElMjBGb3JtYSUyMChCJTIwKiUyMEclMkMpJTIwJTNEJTIwKDglMkMpJTBBYWR2YW50YWdlcyUyMCUzRCUyMChyZXdhcmRzJTIwLSUyMG1lYW5fZ3JvdXBlZF9yZXdhcmRzKSUyMCUyRiUyMChzdGRfZ3JvdXBlZF9yZXdhcmRzJTIwJTJCJTIwMWUtOCk=",highlighted:`<span class="hljs-comment"># Avantaje: Forma (B * G,) = (8,)</span> | |
| advantages = (rewards - mean_grouped_rewards) / (std_grouped_rewards + <span class="hljs-number">1e-8</span>)`,wrap:!1}}),fa=new v({props:{code:"QXZhbnRhamUlM0ElMjB0ZW5zb3IoJTVCJTIwMC44NjU5JTJDJTIwLTAuODY2MCUyQyUyMC0wLjg2NjAlMkMlMjAlMjAwLjg2NTklMkMlMjAtMC44NjYwJTJDJTIwLTAuODY2MCUyQyUyMCUyMDAuODY1OSUyQyUyMCUyMDAuODY1OSU1RCk=",highlighted:"Avantaje: tensor([ 0.8659, -0.8660, -0.8660, 0.8659, -0.8660, -0.8660, 0.8659, 0.8659])",wrap:!1}}),ba=new v({props:{code:"UGVudHJ1JTIwcmV3YXJkXzElMjAlM0QlMjAlNUIxJTJDJTIwMCUyQyUyMDAlMkMlMjAxJTVEJTNBJTBBMSUyMC0lMjAwLjUlMjAlMkYlMjAwLjU3NzQlMjAlRTIlODklODglMjAwLjg2NTklMEEwJTIwLSUyMDAuNSUyMCUyRiUyMDAuNTc3NCUyMCVFMiU4OSU4OCUyMC0wLjg2NjAlMEFQZW50cnUlMjByZXdhcmRfMiUyMCUzRCUyMCU1QjAlMkMlMjAwJTJDJTIwMSUyQyUyMDElNUQlM0ElMjBBY2VsYSVDOCU5OWklMjBtb2RlbC4=",highlighted:`Pentru reward_1 = [1, 0, 0, 1]: | |
| 1 - 0.5 / 0.5774 ≈ 0.8659 | |
| 0 - 0.5 / 0.5774 ≈ -0.8660 | |
| Pentru reward_2 = [0, 0, 1, 1]: Același model.`,wrap:!1}}),Ja=new v({props:{code:"JTIzJTIwRm9ybWElMjAoQiUyMColMjBHJTJDJTIwMSklMjAlM0QlMjAoOCUyQyUyMDEpJTIwcGVudHJ1JTIwYSUyMHNlJTIwcG90cml2aSUyMGN1JTIwZm9ybWElMjBsb2dpdHMlMEFhZHZhbnRhZ2VzJTIwJTNEJTIwYWR2YW50YWdlcy51bnNxdWVlemUoMSk=",highlighted:`<span class="hljs-comment"># Forma (B * G, 1) = (8, 1) pentru a se potrivi cu forma logits</span> | |
| advantages = advantages.unsqueeze(<span class="hljs-number">1</span>)`,wrap:!1}}),Ta=new v({props:{code:"QXZhbnRhamUlM0ElMjB0ZW5zb3IoJTVCJTVCJTIwMC44NjU5JTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCLTAuODY2MCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1Qi0wLjg2NjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUIlMjAwLjg2NTklNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUItMC44NjYwJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCLTAuODY2MCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1QiUyMDAuODY1OSU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1QiUyMDAuODY1OSU1RCU1RCk=",highlighted:`Avantaje: tensor([[ 0.8659], | |
| [-0.8660], | |
| [-0.8660], | |
| [ 0.8659], | |
| [-0.8660], | |
| [-0.8660], | |
| [ 0.8659], | |
| [ 0.8659]])`,wrap:!1}}),xa=new g({props:{title:"3. Actualizarea Politicii",local:"3-actualizarea-politicii",headingTag:"h3"}}),ja=new v({props:{code:"JTIzJTIwQ2FsY3VsZWF6JUM0JTgzJTIwcmFwb3J0dWwlMjBkZSUyMHByb2JhYmlsaXRhdGUlMjAlQzMlQUVudHJlJTIwcG9saXRpY2lsZSUyMG5vaSUyMCVDOCU5OWklMjB2ZWNoaSUwQXJhdGlvJTIwJTNEJTIwdG9yY2guZXhwKCUwQSUyMCUyMCUyMCUyMG5ld19wZXJfdG9rZW5fbG9ncHMlMjAtJTIwcGVyX3Rva2VuX2xvZ3BzJTBBKSUyMCUyMCUyMyUyMEZvcm1hJTNBJTIwKEIqRyUyQyUyMHNlcV9sZW4pJTIwc2VxX2xlbiUyMGVzdGUlMjBsdW5naW1lYSUyMHJlenVsdGF0dWx1aSUyMGFkaWMlQzQlODMlMjBudW0lQzQlODNydWwlMjBkZSUyMHRva2VuLXVyaSUyMGdlbmVyYXRlJTIwZGVjaSUyMGFpY2klMjBwZW50cnUlMjBzaW1wbGl0YXRlJTIwcyVDNCU4MyUyMHByZXN1cHVuZW0lMjBjJUM0JTgzJTIwZXN0ZSUyMDElMjAlMjMlMjAoOCUyQyUyMDEp",highlighted:`<span class="hljs-comment"># Calculează raportul de probabilitate între politicile noi și vechi</span> | |
| ratio = torch.exp( | |
| new_per_token_logps - per_token_logps | |
| ) <span class="hljs-comment"># Forma: (B*G, seq_len) seq_len este lungimea rezultatului adică numărul de token-uri generate deci aici pentru simplitate să presupunem că este 1 # (8, 1)</span>`,wrap:!1}}),Ia=new v({props:{code:"JTIzJTIwRnVuYyVDOCU5QmlhJTIwZGUlMjBUJUM0JTgzaWVyZSUwQWVwcyUyMCUzRCUyMHNlbGYuY2xpcHJhbmdlJTIwJTIwJTIzJTIwZGUlMjBleGVtcGx1JTIwMC4yJTBBcGdfbG9zc2VzMSUyMCUzRCUyMC1hZHZhbnRhZ2VzJTIwKiUyMHJhdGlvJTIwJTIwJTIzJTIwRm9ybWElM0ElMjAoQipHJTJDJTIwc2VxX2xlbiklMjAlMjAlMjMoOCUyQyUyMDEpJTBBcGdfbG9zc2VzMiUyMCUzRCUyMC1hZHZhbnRhZ2VzJTIwKiUyMHRvcmNoLmNsYW1wKCUwQSUyMCUyMCUyMCUyMHJhdGlvJTJDJTIwMS4wJTIwLSUyMGVwcyUyQyUyMDEuMCUyMCUyQiUyMGVwcyUwQSklMjAlMjAlMjMlMjBGb3JtYSUzQSUyMChCKkclMkMlMjBzZXFfbGVuKSUyMCUyMyg4JTJDJTIwMSklMEFwZ19sb3NzX21heCUyMCUzRCUyMHRvcmNoLm1heChwZ19sb3NzZXMxJTJDJTIwcGdfbG9zc2VzMiklMjAlMjAlMjMlMjBGb3JtYSUzQSUyMChCKkclMkMlMjBzZXFfbGVuKSUyMCUyMyg4JTJDJTIwMSklMEElMEElMEElMjMlMjBBY3VtJTIwQ29tYmluJUM0JTgzJTIwY3UlMjBwZW5hbGl0YXRlYSUyMEtMJTIwJTIzJTIwRm9ybWElM0ElMjAoQipHJTJDJTIwc2VxX2xlbiklMjAlMjMoOCUyQyUyMDEpJTBBcGVyX3Rva2VuX2xvc3MlMjAlM0QlMjBwZ19sb3NzX21heCUyMCUyQiUyMHNlbGYuYmV0YSUyMColMjBwZXJfdG9rZW5fa2w=",highlighted:`<span class="hljs-comment"># Funcția de Tăiere</span> | |
| eps = self.cliprange <span class="hljs-comment"># de exemplu 0.2</span> | |
| pg_losses1 = -advantages * ratio <span class="hljs-comment"># Forma: (B*G, seq_len) #(8, 1)</span> | |
| pg_losses2 = -advantages * torch.clamp( | |
| ratio, <span class="hljs-number">1.0</span> - eps, <span class="hljs-number">1.0</span> + eps | |
| ) <span class="hljs-comment"># Forma: (B*G, seq_len) #(8, 1)</span> | |
| pg_loss_max = torch.<span class="hljs-built_in">max</span>(pg_losses1, pg_losses2) <span class="hljs-comment"># Forma: (B*G, seq_len) #(8, 1)</span> | |
| <span class="hljs-comment"># Acum Combină cu penalitatea KL # Forma: (B*G, seq_len) #(8, 1)</span> | |
| per_token_loss = pg_loss_max + self.beta * per_token_kl`,wrap:!1}}),ka=new v({props:{code:"JTIzJTIwRm9ybWElM0ElMjAoQipHJTJDJTIwc2VxX2xlbiklMjAlMjMoOCUyQyUyMDEpJTBBcGVyX3Rva2VuX2tsJTIwJTNEJTIwRi5rbF9kaXYoJTBBJTIwJTIwJTIwJTIwRi5sb2dfc29mdG1heChuZXdfcGVyX3Rva2VuX2xvZ3BzJTJDJTIwZGltJTNELTEpJTJDJTBBJTIwJTIwJTIwJTIwRi5zb2Z0bWF4KHBlcl90b2tlbl9sb2dwcyUyQyUyMGRpbSUzRC0xKSUyQyUwQSUyMCUyMCUyMCUyMHJlZHVjdGlvbiUzRCUyMm5vbmUlMjIlMkMlMEEpLnN1bShkaW0lM0QtMSUyQyUyMGtlZXBkaW0lM0RUcnVlKQ==",highlighted:`<span class="hljs-comment"># Forma: (B*G, seq_len) #(8, 1)</span> | |
| per_token_kl = F.kl_div( | |
| F.log_softmax(new_per_token_logps, dim=-<span class="hljs-number">1</span>), | |
| F.softmax(per_token_logps, dim=-<span class="hljs-number">1</span>), | |
| reduction=<span class="hljs-string">"none"</span>, | |
| ).<span class="hljs-built_in">sum</span>(dim=-<span class="hljs-number">1</span>, keepdim=<span class="hljs-literal">True</span>)`,wrap:!1}}),Ga=new g({props:{title:"Rezumat și Pași Următori",local:"rezumat-și-pași-următori",headingTag:"h2"}}),Xa=new g({props:{title:"Referințe",local:"referințe",headingTag:"h2"}}),La=new Bp({props:{source:"https://github.com/huggingface/course/blob/main/chapters/rum/chapter12/3a.mdx"}}),{c(){M=p("meta"),z=l(),f=p("p"),Wa=l(),r(j.$$.fragment),le=l(),r($.$$.fragment),ne=l(),_=p("p"),_.textContent=Nl,pe=l(),I=p("p"),I.textContent=Fl,ie=l(),A=p("p"),A.textContent=Yl,me=l(),k=p("p"),k.textContent=Ol,re=l(),R=p("p"),R.innerHTML=ql,ce=l(),G=p("p"),G.textContent=Kl,oe=l(),r(B.$$.fragment),ue=l(),Q=p("p"),Q.textContent=sn,he=l(),P=p("p"),P.textContent=an,de=l(),r(Z.$$.fragment),ge=l(),V=p("p"),V.textContent=en,ye=l(),X=p("p"),X.innerHTML=tn,Me=l(),r(E.$$.fragment),ve=l(),L=p("p"),L.textContent=ln,fe=l(),D=p("ul"),D.innerHTML=nn,we=l(),S=p("p"),S.textContent=pn,be=l(),r(W.$$.fragment),$e=l(),H=p("p"),H.textContent=mn,Je=l(),r(N.$$.fragment),Ue=l(),F=p("p"),F.textContent=rn,Te=l(),Y=p("p"),Y.innerHTML=cn,Ce=l(),r(O.$$.fragment),xe=l(),q=p("p"),Gl=C(`Ideea cheie a GRPO este că nu avem nevoie de măsuri absolute ale calității - putem compara rezultatele în cadrul aceluiași grup. Aceasta se face folosind standardizarea: | |
| `),ze=new Da(!1),je=l(),r(K.$$.fragment),_e=l(),ss=p("p"),ss.textContent=on,Ie=l(),as=p("ul"),as.innerHTML=un,Ae=l(),r(es.$$.fragment),ke=l(),ts=p("p"),ts.textContent=hn,Re=l(),ls=p("p"),ls.textContent=dn,Ge=l(),ns=p("p"),ns.textContent=gn,Be=l(),ps=p("p"),ps.textContent=yn,Qe=l(),r(is.$$.fragment),Pe=l(),ms=p("p"),ms.textContent=Mn,Ze=l(),rs=p("p"),Bl=C(`Funcția țintă pentru actualizarea politicii este: | |
| `),Ve=new Da(!1),Xe=l(),cs=p("p"),cs.textContent=vn,Ee=l(),r(os.$$.fragment),Le=l(),us=p("p"),us.textContent=fn,De=l(),r(hs.$$.fragment),Se=l(),ds=p("p"),ds.textContent=wn,We=l(),gs=p("p"),gs.innerHTML=bn,He=l(),ys=p("p"),ys.textContent=$n,Ne=l(),r(Ms.$$.fragment),Fe=l(),vs=p("ul"),vs.innerHTML=Jn,Ye=l(),fs=p("p"),fs.textContent=Un,Oe=l(),r(ws.$$.fragment),qe=l(),bs=p("p"),bs.textContent=Tn,Ke=l(),$s=p("p"),$s.innerHTML=Cn,st=l(),Js=p("p"),Js.textContent=xn,at=l(),r(Us.$$.fragment),et=l(),Ts=p("p"),Ts.textContent=zn,tt=l(),Cs=p("ul"),Cs.innerHTML=jn,lt=l(),r(xs.$$.fragment),nt=l(),zs=p("ul"),zs.innerHTML=_n,pt=l(),js=p("p"),js.textContent=In,it=l(),r(_s.$$.fragment),mt=l(),Is=p("p"),Is.textContent=An,rt=l(),As=p("p"),As.innerHTML=kn,ct=l(),ks=p("p"),ks.innerHTML=Rn,ot=l(),r(Rs.$$.fragment),ut=l(),Gs=p("ul"),Gs.innerHTML=Gn,ht=l(),r(Bs.$$.fragment),dt=l(),Qs=p("p"),Qs.textContent=Bn,gt=l(),J=p("p"),Ql=C("Să ne amintim că distanța KL este definită după cum urmează:"),yt=new Da(!1),Mt=C(` | |
| În RLHF, cele două distribuții de interes sunt adesea distribuția versiunii noului model, P(x), și o distribuție a politicii de referință, Q(x).`),vt=l(),r(Ps.$$.fragment),ft=l(),Zs=p("p"),Zs.textContent=Qn,wt=l(),Vs=p("ul"),Vs.innerHTML=Pn,bt=l(),Xs=p("p"),Xs.textContent=Zn,$t=l(),r(Es.$$.fragment),Jt=l(),Ls=p("p"),Ls.textContent=Vn,Ut=l(),r(Ds.$$.fragment),Tt=l(),Ha=p("p"),Ct=new Da(!1),xt=l(),r(Ss.$$.fragment),zt=l(),Ws=p("p"),Ws.textContent=Xn,jt=l(),Hs=p("p"),Pl=C(`Generează $(G = 8)$ răspunsuri, $4$ dintre care sunt răspunsul corect ($14, \\text{recompensă=} 1$) și $4$ incorecte $\\text{(recompensă= 0)}$, Prin urmare: | |
| `),_t=new Da(!1),It=l(),r(Ns.$$.fragment),At=l(),Fs=p("p"),Fs.textContent=En,kt=l(),b=p("ul"),Na=p("li"),Zl=C("Media Grupului: "),Rt=new Da(!1),Vl=l(),Fa=p("li"),Fa.textContent=Ln,Xl=l(),Ya=p("li"),Ya.innerHTML=Dn,Gt=l(),r(Ys.$$.fragment),Bt=l(),Os=p("p"),Os.textContent=Sn,Qt=l(),U=p("ul"),T=p("li"),El=C("Presupunând că probabilitatea vechii politici ($\\pi"),Oa=p("em"),Oa.textContent=Wn,Ll=C("{old}}$) pentru un rezultat corect $o_1$ este $0.5$ și noua politică o crește la $0.7$ atunci:"),Pt=new Da(!1),Dl=l(),qa=p("li"),qa.textContent=Hn,Zt=l(),qs=p("p"),qs.textContent=Nn,Vt=l(),r(Ks.$$.fragment),Xt=l(),sa=p("p"),sa.textContent=Fn,Et=l(),r(aa.$$.fragment),Lt=l(),ea=p("p"),ea.textContent=Yn,Dt=l(),r(ta.$$.fragment),St=l(),la=p("p"),la.textContent=On,Wt=l(),r(na.$$.fragment),Ht=l(),r(pa.$$.fragment),Nt=l(),ia=p("p"),ia.textContent=qn,Ft=l(),ma=p("p"),ma.innerHTML=Kn,Yt=l(),ra=p("ul"),ra.innerHTML=sp,Ot=l(),ca=p("p"),ca.textContent=ap,qt=l(),r(oa.$$.fragment),Kt=l(),ua=p("p"),ua.textContent=ep,sl=l(),r(ha.$$.fragment),al=l(),da=p("p"),da.textContent=tp,el=l(),r(ga.$$.fragment),tl=l(),ya=p("p"),ya.textContent=lp,ll=l(),r(Ma.$$.fragment),nl=l(),va=p("p"),va.textContent=np,pl=l(),r(fa.$$.fragment),il=l(),wa=p("p"),wa.textContent=pp,ml=l(),r(ba.$$.fragment),rl=l(),$a=p("p"),$a.innerHTML=ip,cl=l(),r(Ja.$$.fragment),ol=l(),Ua=p("p"),Ua.textContent=mp,ul=l(),r(Ta.$$.fragment),hl=l(),Ca=p("p"),Ca.textContent=rp,dl=l(),r(xa.$$.fragment),gl=l(),za=p("p"),za.textContent=cp,yl=l(),r(ja.$$.fragment),Ml=l(),_a=p("p"),_a.innerHTML=op,vl=l(),r(Ia.$$.fragment),fl=l(),Aa=p("p"),Aa.innerHTML=up,wl=l(),r(ka.$$.fragment),bl=l(),Ra=p("p"),Ra.innerHTML=hp,$l=l(),r(Ga.$$.fragment),Jl=l(),Ba=p("p"),Ba.textContent=dp,Ul=l(),Qa=p("ol"),Qa.innerHTML=gp,Tl=l(),Pa=p("p"),Pa.textContent=yp,Cl=l(),Za=p("p"),Za.textContent=Mp,xl=l(),Va=p("p"),Va.textContent=vp,zl=l(),r(Xa.$$.fragment),jl=l(),Ea=p("ol"),Ea.innerHTML=fp,_l=l(),r(La.$$.fragment),Il=l(),ae=p("p"),this.h()},l(s){const a=Rp("svelte-u9bgzb",document.head);M=i(a,"META",{name:!0,content:!0}),a.forEach(e),z=n(s),f=i(s,"P",{}),w(f).forEach(e),Wa=n(s),c(j.$$.fragment,s),le=n(s),c($.$$.fragment,s),ne=n(s),_=i(s,"P",{"data-svelte-h":!0}),m(_)!=="svelte-a8nopu"&&(_.textContent=Nl),pe=n(s),I=i(s,"P",{"data-svelte-h":!0}),m(I)!=="svelte-5h4nn6"&&(I.textContent=Fl),ie=n(s),A=i(s,"P",{"data-svelte-h":!0}),m(A)!=="svelte-ngln34"&&(A.textContent=Yl),me=n(s),k=i(s,"P",{"data-svelte-h":!0}),m(k)!=="svelte-1cq9ocq"&&(k.textContent=Ol),re=n(s),R=i(s,"P",{"data-svelte-h":!0}),m(R)!=="svelte-grajek"&&(R.innerHTML=ql),ce=n(s),G=i(s,"P",{"data-svelte-h":!0}),m(G)!=="svelte-183mr8p"&&(G.textContent=Kl),oe=n(s),c(B.$$.fragment,s),ue=n(s),Q=i(s,"P",{"data-svelte-h":!0}),m(Q)!=="svelte-o4t79o"&&(Q.textContent=sn),he=n(s),P=i(s,"P",{"data-svelte-h":!0}),m(P)!=="svelte-1mq2b68"&&(P.textContent=an),de=n(s),c(Z.$$.fragment,s),ge=n(s),V=i(s,"P",{"data-svelte-h":!0}),m(V)!=="svelte-1333z4y"&&(V.textContent=en),ye=n(s),X=i(s,"P",{"data-svelte-h":!0}),m(X)!=="svelte-gv96n3"&&(X.innerHTML=tn),Me=n(s),c(E.$$.fragment,s),ve=n(s),L=i(s,"P",{"data-svelte-h":!0}),m(L)!=="svelte-1bc8nd7"&&(L.textContent=ln),fe=n(s),D=i(s,"UL",{"data-svelte-h":!0}),m(D)!=="svelte-16dcrvi"&&(D.innerHTML=nn),we=n(s),S=i(s,"P",{"data-svelte-h":!0}),m(S)!=="svelte-w6fns5"&&(S.textContent=pn),be=n(s),c(W.$$.fragment,s),$e=n(s),H=i(s,"P",{"data-svelte-h":!0}),m(H)!=="svelte-1powh5y"&&(H.textContent=mn),Je=n(s),c(N.$$.fragment,s),Ue=n(s),F=i(s,"P",{"data-svelte-h":!0}),m(F)!=="svelte-ahcfcq"&&(F.textContent=rn),Te=n(s),Y=i(s,"P",{"data-svelte-h":!0}),m(Y)!=="svelte-j4v8ca"&&(Y.innerHTML=cn),Ce=n(s),c(O.$$.fragment,s),xe=n(s),q=i(s,"P",{});var Ka=w(q);Gl=x(Ka,`Ideea cheie a GRPO este că nu avem nevoie de măsuri absolute ale calității - putem compara rezultatele în cadrul aceluiași grup. Aceasta se face folosind standardizarea: | |
| `),ze=Sa(Ka,!1),Ka.forEach(e),je=n(s),c(K.$$.fragment,s),_e=n(s),ss=i(s,"P",{"data-svelte-h":!0}),m(ss)!=="svelte-1dbhl51"&&(ss.textContent=on),Ie=n(s),as=i(s,"UL",{"data-svelte-h":!0}),m(as)!=="svelte-1dz26ga"&&(as.innerHTML=un),Ae=n(s),c(es.$$.fragment,s),ke=n(s),ts=i(s,"P",{"data-svelte-h":!0}),m(ts)!=="svelte-1tzkh94"&&(ts.textContent=hn),Re=n(s),ls=i(s,"P",{"data-svelte-h":!0}),m(ls)!=="svelte-1cseuus"&&(ls.textContent=dn),Ge=n(s),ns=i(s,"P",{"data-svelte-h":!0}),m(ns)!=="svelte-z9skjy"&&(ns.textContent=gn),Be=n(s),ps=i(s,"P",{"data-svelte-h":!0}),m(ps)!=="svelte-1j510qt"&&(ps.textContent=yn),Qe=n(s),c(is.$$.fragment,s),Pe=n(s),ms=i(s,"P",{"data-svelte-h":!0}),m(ms)!=="svelte-ruqsaq"&&(ms.textContent=Mn),Ze=n(s),rs=i(s,"P",{});var Sl=w(rs);Bl=x(Sl,`Funcția țintă pentru actualizarea politicii este: | |
| `),Ve=Sa(Sl,!1),Sl.forEach(e),Xe=n(s),cs=i(s,"P",{"data-svelte-h":!0}),m(cs)!=="svelte-1box5d8"&&(cs.textContent=vn),Ee=n(s),c(os.$$.fragment,s),Le=n(s),us=i(s,"P",{"data-svelte-h":!0}),m(us)!=="svelte-1trvisp"&&(us.textContent=fn),De=n(s),c(hs.$$.fragment,s),Se=n(s),ds=i(s,"P",{"data-svelte-h":!0}),m(ds)!=="svelte-yq5cee"&&(ds.textContent=wn),We=n(s),gs=i(s,"P",{"data-svelte-h":!0}),m(gs)!=="svelte-1089qz0"&&(gs.innerHTML=bn),He=n(s),ys=i(s,"P",{"data-svelte-h":!0}),m(ys)!=="svelte-ku2ikr"&&(ys.textContent=$n),Ne=n(s),c(Ms.$$.fragment,s),Fe=n(s),vs=i(s,"UL",{"data-svelte-h":!0}),m(vs)!=="svelte-o0khd3"&&(vs.innerHTML=Jn),Ye=n(s),fs=i(s,"P",{"data-svelte-h":!0}),m(fs)!=="svelte-1qnq02v"&&(fs.textContent=Un),Oe=n(s),c(ws.$$.fragment,s),qe=n(s),bs=i(s,"P",{"data-svelte-h":!0}),m(bs)!=="svelte-17prawn"&&(bs.textContent=Tn),Ke=n(s),$s=i(s,"P",{"data-svelte-h":!0}),m($s)!=="svelte-csgxyq"&&($s.innerHTML=Cn),st=n(s),Js=i(s,"P",{"data-svelte-h":!0}),m(Js)!=="svelte-1rsi9fp"&&(Js.textContent=xn),at=n(s),c(Us.$$.fragment,s),et=n(s),Ts=i(s,"P",{"data-svelte-h":!0}),m(Ts)!=="svelte-bjglm8"&&(Ts.textContent=zn),tt=n(s),Cs=i(s,"UL",{"data-svelte-h":!0}),m(Cs)!=="svelte-5yqt2b"&&(Cs.innerHTML=jn),lt=n(s),c(xs.$$.fragment,s),nt=n(s),zs=i(s,"UL",{"data-svelte-h":!0}),m(zs)!=="svelte-1130437"&&(zs.innerHTML=_n),pt=n(s),js=i(s,"P",{"data-svelte-h":!0}),m(js)!=="svelte-1duj7j2"&&(js.textContent=In),it=n(s),c(_s.$$.fragment,s),mt=n(s),Is=i(s,"P",{"data-svelte-h":!0}),m(Is)!=="svelte-3n8uqr"&&(Is.textContent=An),rt=n(s),As=i(s,"P",{"data-svelte-h":!0}),m(As)!=="svelte-zkgyxi"&&(As.innerHTML=kn),ct=n(s),ks=i(s,"P",{"data-svelte-h":!0}),m(ks)!=="svelte-1q1mnal"&&(ks.innerHTML=Rn),ot=n(s),c(Rs.$$.fragment,s),ut=n(s),Gs=i(s,"UL",{"data-svelte-h":!0}),m(Gs)!=="svelte-aeq2nv"&&(Gs.innerHTML=Gn),ht=n(s),c(Bs.$$.fragment,s),dt=n(s),Qs=i(s,"P",{"data-svelte-h":!0}),m(Qs)!=="svelte-cn7jji"&&(Qs.textContent=Bn),gt=n(s),J=i(s,"P",{});var kl=w(J);Ql=x(kl,"Să ne amintim că distanța KL este definită după cum urmează:"),yt=Sa(kl,!1),Mt=x(kl,` | |
| În RLHF, cele două distribuții de interes sunt adesea distribuția versiunii noului model, P(x), și o distribuție a politicii de referință, Q(x).`),kl.forEach(e),vt=n(s),c(Ps.$$.fragment,s),ft=n(s),Zs=i(s,"P",{"data-svelte-h":!0}),m(Zs)!=="svelte-xvmook"&&(Zs.textContent=Qn),wt=n(s),Vs=i(s,"UL",{"data-svelte-h":!0}),m(Vs)!=="svelte-kia2ga"&&(Vs.innerHTML=Pn),bt=n(s),Xs=i(s,"P",{"data-svelte-h":!0}),m(Xs)!=="svelte-cvqt1z"&&(Xs.textContent=Zn),$t=n(s),c(Es.$$.fragment,s),Jt=n(s),Ls=i(s,"P",{"data-svelte-h":!0}),m(Ls)!=="svelte-1fswzuo"&&(Ls.textContent=Vn),Ut=n(s),c(Ds.$$.fragment,s),Tt=n(s),Ha=i(s,"P",{});var wp=w(Ha);Ct=Sa(wp,!1),wp.forEach(e),xt=n(s),c(Ss.$$.fragment,s),zt=n(s),Ws=i(s,"P",{"data-svelte-h":!0}),m(Ws)!=="svelte-1gd6e76"&&(Ws.textContent=Xn),jt=n(s),Hs=i(s,"P",{});var Wl=w(Hs);Pl=x(Wl,`Generează $(G = 8)$ răspunsuri, $4$ dintre care sunt răspunsul corect ($14, \\text{recompensă=} 1$) și $4$ incorecte $\\text{(recompensă= 0)}$, Prin urmare: | |
| `),_t=Sa(Wl,!1),Wl.forEach(e),It=n(s),c(Ns.$$.fragment,s),At=n(s),Fs=i(s,"P",{"data-svelte-h":!0}),m(Fs)!=="svelte-17wlw5b"&&(Fs.textContent=En),kt=n(s),b=i(s,"UL",{});var se=w(b);Na=i(se,"LI",{});var Hl=w(Na);Zl=x(Hl,"Media Grupului: "),Rt=Sa(Hl,!1),Hl.forEach(e),Vl=n(se),Fa=i(se,"LI",{"data-svelte-h":!0}),m(Fa)!=="svelte-11g5ftk"&&(Fa.textContent=Ln),Xl=n(se),Ya=i(se,"LI",{"data-svelte-h":!0}),m(Ya)!=="svelte-doylt2"&&(Ya.innerHTML=Dn),se.forEach(e),Gt=n(s),c(Ys.$$.fragment,s),Bt=n(s),Os=i(s,"P",{"data-svelte-h":!0}),m(Os)!=="svelte-1an3ncn"&&(Os.textContent=Sn),Qt=n(s),U=i(s,"UL",{});var Rl=w(U);T=i(Rl,"LI",{});var ee=w(T);El=x(ee,"Presupunând că probabilitatea vechii politici ($\\pi"),Oa=i(ee,"EM",{"data-svelte-h":!0}),m(Oa)!=="svelte-19ssaec"&&(Oa.textContent=Wn),Ll=x(ee,"{old}}$) pentru un rezultat corect $o_1$ este $0.5$ și noua politică o crește la $0.7$ atunci:"),Pt=Sa(ee,!1),ee.forEach(e),Dl=n(Rl),qa=i(Rl,"LI",{"data-svelte-h":!0}),m(qa)!=="svelte-1jyckmz"&&(qa.textContent=Hn),Rl.forEach(e),Zt=n(s),qs=i(s,"P",{"data-svelte-h":!0}),m(qs)!=="svelte-1hm03k5"&&(qs.textContent=Nn),Vt=n(s),c(Ks.$$.fragment,s),Xt=n(s),sa=i(s,"P",{"data-svelte-h":!0}),m(sa)!=="svelte-ftndne"&&(sa.textContent=Fn),Et=n(s),c(aa.$$.fragment,s),Lt=n(s),ea=i(s,"P",{"data-svelte-h":!0}),m(ea)!=="svelte-1ton2ix"&&(ea.textContent=Yn),Dt=n(s),c(ta.$$.fragment,s),St=n(s),la=i(s,"P",{"data-svelte-h":!0}),m(la)!=="svelte-8p3zcf"&&(la.textContent=On),Wt=n(s),c(na.$$.fragment,s),Ht=n(s),c(pa.$$.fragment,s),Nt=n(s),ia=i(s,"P",{"data-svelte-h":!0}),m(ia)!=="svelte-1xtsdok"&&(ia.textContent=qn),Ft=n(s),ma=i(s,"P",{"data-svelte-h":!0}),m(ma)!=="svelte-dc3rvk"&&(ma.innerHTML=Kn),Yt=n(s),ra=i(s,"UL",{"data-svelte-h":!0}),m(ra)!=="svelte-1ts0tfj"&&(ra.innerHTML=sp),Ot=n(s),ca=i(s,"P",{"data-svelte-h":!0}),m(ca)!=="svelte-1ivder3"&&(ca.textContent=ap),qt=n(s),c(oa.$$.fragment,s),Kt=n(s),ua=i(s,"P",{"data-svelte-h":!0}),m(ua)!=="svelte-1d96x8n"&&(ua.textContent=ep),sl=n(s),c(ha.$$.fragment,s),al=n(s),da=i(s,"P",{"data-svelte-h":!0}),m(da)!=="svelte-ds8ok1"&&(da.textContent=tp),el=n(s),c(ga.$$.fragment,s),tl=n(s),ya=i(s,"P",{"data-svelte-h":!0}),m(ya)!=="svelte-1wq4gtx"&&(ya.textContent=lp),ll=n(s),c(Ma.$$.fragment,s),nl=n(s),va=i(s,"P",{"data-svelte-h":!0}),m(va)!=="svelte-ds8ok1"&&(va.textContent=np),pl=n(s),c(fa.$$.fragment,s),il=n(s),wa=i(s,"P",{"data-svelte-h":!0}),m(wa)!=="svelte-5joddo"&&(wa.textContent=pp),ml=n(s),c(ba.$$.fragment,s),rl=n(s),$a=i(s,"P",{"data-svelte-h":!0}),m($a)!=="svelte-3ix9wi"&&($a.innerHTML=ip),cl=n(s),c(Ja.$$.fragment,s),ol=n(s),Ua=i(s,"P",{"data-svelte-h":!0}),m(Ua)!=="svelte-1qni2jw"&&(Ua.textContent=mp),ul=n(s),c(Ta.$$.fragment,s),hl=n(s),Ca=i(s,"P",{"data-svelte-h":!0}),m(Ca)!=="svelte-1xdga66"&&(Ca.textContent=rp),dl=n(s),c(xa.$$.fragment,s),gl=n(s),za=i(s,"P",{"data-svelte-h":!0}),m(za)!=="svelte-anpd45"&&(za.textContent=cp),yl=n(s),c(ja.$$.fragment,s),Ml=n(s),_a=i(s,"P",{"data-svelte-h":!0}),m(_a)!=="svelte-zbd9id"&&(_a.innerHTML=op),vl=n(s),c(Ia.$$.fragment,s),fl=n(s),Aa=i(s,"P",{"data-svelte-h":!0}),m(Aa)!=="svelte-wqakk1"&&(Aa.innerHTML=up),wl=n(s),c(ka.$$.fragment,s),bl=n(s),Ra=i(s,"P",{"data-svelte-h":!0}),m(Ra)!=="svelte-1wguwjn"&&(Ra.innerHTML=hp),$l=n(s),c(Ga.$$.fragment,s),Jl=n(s),Ba=i(s,"P",{"data-svelte-h":!0}),m(Ba)!=="svelte-9qtprz"&&(Ba.textContent=dp),Ul=n(s),Qa=i(s,"OL",{"data-svelte-h":!0}),m(Qa)!=="svelte-16pwkt8"&&(Qa.innerHTML=gp),Tl=n(s),Pa=i(s,"P",{"data-svelte-h":!0}),m(Pa)!=="svelte-1fln7ar"&&(Pa.textContent=yp),Cl=n(s),Za=i(s,"P",{"data-svelte-h":!0}),m(Za)!=="svelte-1mu4b8k"&&(Za.textContent=Mp),xl=n(s),Va=i(s,"P",{"data-svelte-h":!0}),m(Va)!=="svelte-1v7tjb"&&(Va.textContent=vp),zl=n(s),c(Xa.$$.fragment,s),jl=n(s),Ea=i(s,"OL",{"data-svelte-h":!0}),m(Ea)!=="svelte-1hdfyr"&&(Ea.innerHTML=fp),_l=n(s),c(La.$$.fragment,s),Il=n(s),ae=i(s,"P",{}),w(ae).forEach(e),this.h()},h(){bp(M,"name","hf:doc:metadata"),bp(M,"content",Zp),ze.a=null,Ve.a=null,yt.a=Mt,Ct.a=null,_t.a=null,Rt.a=null,Pt.a=null},m(s,a){y(document.head,M),t(s,z,a),t(s,f,a),t(s,Wa,a),o(j,s,a),t(s,le,a),o($,s,a),t(s,ne,a),t(s,_,a),t(s,pe,a),t(s,I,a),t(s,ie,a),t(s,A,a),t(s,me,a),t(s,k,a),t(s,re,a),t(s,R,a),t(s,ce,a),t(s,G,a),t(s,oe,a),o(B,s,a),t(s,ue,a),t(s,Q,a),t(s,he,a),t(s,P,a),t(s,de,a),o(Z,s,a),t(s,ge,a),t(s,V,a),t(s,ye,a),t(s,X,a),t(s,Me,a),o(E,s,a),t(s,ve,a),t(s,L,a),t(s,fe,a),t(s,D,a),t(s,we,a),t(s,S,a),t(s,be,a),o(W,s,a),t(s,$e,a),t(s,H,a),t(s,Je,a),o(N,s,a),t(s,Ue,a),t(s,F,a),t(s,Te,a),t(s,Y,a),t(s,Ce,a),o(O,s,a),t(s,xe,a),t(s,q,a),y(q,Gl),ze.m($p,q),t(s,je,a),o(K,s,a),t(s,_e,a),t(s,ss,a),t(s,Ie,a),t(s,as,a),t(s,Ae,a),o(es,s,a),t(s,ke,a),t(s,ts,a),t(s,Re,a),t(s,ls,a),t(s,Ge,a),t(s,ns,a),t(s,Be,a),t(s,ps,a),t(s,Qe,a),o(is,s,a),t(s,Pe,a),t(s,ms,a),t(s,Ze,a),t(s,rs,a),y(rs,Bl),Ve.m(Jp,rs),t(s,Xe,a),t(s,cs,a),t(s,Ee,a),o(os,s,a),t(s,Le,a),t(s,us,a),t(s,De,a),o(hs,s,a),t(s,Se,a),t(s,ds,a),t(s,We,a),t(s,gs,a),t(s,He,a),t(s,ys,a),t(s,Ne,a),o(Ms,s,a),t(s,Fe,a),t(s,vs,a),t(s,Ye,a),t(s,fs,a),t(s,Oe,a),o(ws,s,a),t(s,qe,a),t(s,bs,a),t(s,Ke,a),t(s,$s,a),t(s,st,a),t(s,Js,a),t(s,at,a),o(Us,s,a),t(s,et,a),t(s,Ts,a),t(s,tt,a),t(s,Cs,a),t(s,lt,a),o(xs,s,a),t(s,nt,a),t(s,zs,a),t(s,pt,a),t(s,js,a),t(s,it,a),o(_s,s,a),t(s,mt,a),t(s,Is,a),t(s,rt,a),t(s,As,a),t(s,ct,a),t(s,ks,a),t(s,ot,a),o(Rs,s,a),t(s,ut,a),t(s,Gs,a),t(s,ht,a),o(Bs,s,a),t(s,dt,a),t(s,Qs,a),t(s,gt,a),t(s,J,a),y(J,Ql),yt.m(Up,J),y(J,Mt),t(s,vt,a),o(Ps,s,a),t(s,ft,a),t(s,Zs,a),t(s,wt,a),t(s,Vs,a),t(s,bt,a),t(s,Xs,a),t(s,$t,a),o(Es,s,a),t(s,Jt,a),t(s,Ls,a),t(s,Ut,a),o(Ds,s,a),t(s,Tt,a),t(s,Ha,a),Ct.m(Tp,Ha),t(s,xt,a),o(Ss,s,a),t(s,zt,a),t(s,Ws,a),t(s,jt,a),t(s,Hs,a),y(Hs,Pl),_t.m(Cp,Hs),t(s,It,a),o(Ns,s,a),t(s,At,a),t(s,Fs,a),t(s,kt,a),t(s,b,a),y(b,Na),y(Na,Zl),Rt.m(xp,Na),y(b,Vl),y(b,Fa),y(b,Xl),y(b,Ya),t(s,Gt,a),o(Ys,s,a),t(s,Bt,a),t(s,Os,a),t(s,Qt,a),t(s,U,a),y(U,T),y(T,El),y(T,Oa),y(T,Ll),Pt.m(zp,T),y(U,Dl),y(U,qa),t(s,Zt,a),t(s,qs,a),t(s,Vt,a),o(Ks,s,a),t(s,Xt,a),t(s,sa,a),t(s,Et,a),o(aa,s,a),t(s,Lt,a),t(s,ea,a),t(s,Dt,a),o(ta,s,a),t(s,St,a),t(s,la,a),t(s,Wt,a),o(na,s,a),t(s,Ht,a),o(pa,s,a),t(s,Nt,a),t(s,ia,a),t(s,Ft,a),t(s,ma,a),t(s,Yt,a),t(s,ra,a),t(s,Ot,a),t(s,ca,a),t(s,qt,a),o(oa,s,a),t(s,Kt,a),t(s,ua,a),t(s,sl,a),o(ha,s,a),t(s,al,a),t(s,da,a),t(s,el,a),o(ga,s,a),t(s,tl,a),t(s,ya,a),t(s,ll,a),o(Ma,s,a),t(s,nl,a),t(s,va,a),t(s,pl,a),o(fa,s,a),t(s,il,a),t(s,wa,a),t(s,ml,a),o(ba,s,a),t(s,rl,a),t(s,$a,a),t(s,cl,a),o(Ja,s,a),t(s,ol,a),t(s,Ua,a),t(s,ul,a),o(Ta,s,a),t(s,hl,a),t(s,Ca,a),t(s,dl,a),o(xa,s,a),t(s,gl,a),t(s,za,a),t(s,yl,a),o(ja,s,a),t(s,Ml,a),t(s,_a,a),t(s,vl,a),o(Ia,s,a),t(s,fl,a),t(s,Aa,a),t(s,wl,a),o(ka,s,a),t(s,bl,a),t(s,Ra,a),t(s,$l,a),o(Ga,s,a),t(s,Jl,a),t(s,Ba,a),t(s,Ul,a),t(s,Qa,a),t(s,Tl,a),t(s,Pa,a),t(s,Cl,a),t(s,Za,a),t(s,xl,a),t(s,Va,a),t(s,zl,a),o(Xa,s,a),t(s,jl,a),t(s,Ea,a),t(s,_l,a),o(La,s,a),t(s,Il,a),t(s,ae,a),Al=!0},p(s,[a]){const Ka={};a&2&&(Ka.$$scope={dirty:a,ctx:s}),$.$set(Ka)},i(s){Al||(u(j.$$.fragment,s),u($.$$.fragment,s),u(B.$$.fragment,s),u(Z.$$.fragment,s),u(E.$$.fragment,s),u(W.$$.fragment,s),u(N.$$.fragment,s),u(O.$$.fragment,s),u(K.$$.fragment,s),u(es.$$.fragment,s),u(is.$$.fragment,s),u(os.$$.fragment,s),u(hs.$$.fragment,s),u(Ms.$$.fragment,s),u(ws.$$.fragment,s),u(Us.$$.fragment,s),u(xs.$$.fragment,s),u(_s.$$.fragment,s),u(Rs.$$.fragment,s),u(Bs.$$.fragment,s),u(Ps.$$.fragment,s),u(Es.$$.fragment,s),u(Ds.$$.fragment,s),u(Ss.$$.fragment,s),u(Ns.$$.fragment,s),u(Ys.$$.fragment,s),u(Ks.$$.fragment,s),u(aa.$$.fragment,s),u(ta.$$.fragment,s),u(na.$$.fragment,s),u(pa.$$.fragment,s),u(oa.$$.fragment,s),u(ha.$$.fragment,s),u(ga.$$.fragment,s),u(Ma.$$.fragment,s),u(fa.$$.fragment,s),u(ba.$$.fragment,s),u(Ja.$$.fragment,s),u(Ta.$$.fragment,s),u(xa.$$.fragment,s),u(ja.$$.fragment,s),u(Ia.$$.fragment,s),u(ka.$$.fragment,s),u(Ga.$$.fragment,s),u(Xa.$$.fragment,s),u(La.$$.fragment,s),Al=!0)},o(s){h(j.$$.fragment,s),h($.$$.fragment,s),h(B.$$.fragment,s),h(Z.$$.fragment,s),h(E.$$.fragment,s),h(W.$$.fragment,s),h(N.$$.fragment,s),h(O.$$.fragment,s),h(K.$$.fragment,s),h(es.$$.fragment,s),h(is.$$.fragment,s),h(os.$$.fragment,s),h(hs.$$.fragment,s),h(Ms.$$.fragment,s),h(ws.$$.fragment,s),h(Us.$$.fragment,s),h(xs.$$.fragment,s),h(_s.$$.fragment,s),h(Rs.$$.fragment,s),h(Bs.$$.fragment,s),h(Ps.$$.fragment,s),h(Es.$$.fragment,s),h(Ds.$$.fragment,s),h(Ss.$$.fragment,s),h(Ns.$$.fragment,s),h(Ys.$$.fragment,s),h(Ks.$$.fragment,s),h(aa.$$.fragment,s),h(ta.$$.fragment,s),h(na.$$.fragment,s),h(pa.$$.fragment,s),h(oa.$$.fragment,s),h(ha.$$.fragment,s),h(ga.$$.fragment,s),h(Ma.$$.fragment,s),h(fa.$$.fragment,s),h(ba.$$.fragment,s),h(Ja.$$.fragment,s),h(Ta.$$.fragment,s),h(xa.$$.fragment,s),h(ja.$$.fragment,s),h(Ia.$$.fragment,s),h(ka.$$.fragment,s),h(Ga.$$.fragment,s),h(Xa.$$.fragment,s),h(La.$$.fragment,s),Al=!1},d(s){s&&(e(z),e(f),e(Wa),e(le),e(ne),e(_),e(pe),e(I),e(ie),e(A),e(me),e(k),e(re),e(R),e(ce),e(G),e(oe),e(ue),e(Q),e(he),e(P),e(de),e(ge),e(V),e(ye),e(X),e(Me),e(ve),e(L),e(fe),e(D),e(we),e(S),e(be),e($e),e(H),e(Je),e(Ue),e(F),e(Te),e(Y),e(Ce),e(xe),e(q),e(je),e(_e),e(ss),e(Ie),e(as),e(Ae),e(ke),e(ts),e(Re),e(ls),e(Ge),e(ns),e(Be),e(ps),e(Qe),e(Pe),e(ms),e(Ze),e(rs),e(Xe),e(cs),e(Ee),e(Le),e(us),e(De),e(Se),e(ds),e(We),e(gs),e(He),e(ys),e(Ne),e(Fe),e(vs),e(Ye),e(fs),e(Oe),e(qe),e(bs),e(Ke),e($s),e(st),e(Js),e(at),e(et),e(Ts),e(tt),e(Cs),e(lt),e(nt),e(zs),e(pt),e(js),e(it),e(mt),e(Is),e(rt),e(As),e(ct),e(ks),e(ot),e(ut),e(Gs),e(ht),e(dt),e(Qs),e(gt),e(J),e(vt),e(ft),e(Zs),e(wt),e(Vs),e(bt),e(Xs),e($t),e(Jt),e(Ls),e(Ut),e(Tt),e(Ha),e(xt),e(zt),e(Ws),e(jt),e(Hs),e(It),e(At),e(Fs),e(kt),e(b),e(Gt),e(Bt),e(Os),e(Qt),e(U),e(Zt),e(qs),e(Vt),e(Xt),e(sa),e(Et),e(Lt),e(ea),e(Dt),e(St),e(la),e(Wt),e(Ht),e(Nt),e(ia),e(Ft),e(ma),e(Yt),e(ra),e(Ot),e(ca),e(qt),e(Kt),e(ua),e(sl),e(al),e(da),e(el),e(tl),e(ya),e(ll),e(nl),e(va),e(pl),e(il),e(wa),e(ml),e(rl),e($a),e(cl),e(ol),e(Ua),e(ul),e(hl),e(Ca),e(dl),e(gl),e(za),e(yl),e(Ml),e(_a),e(vl),e(fl),e(Aa),e(wl),e(bl),e(Ra),e($l),e(Jl),e(Ba),e(Ul),e(Qa),e(Tl),e(Pa),e(Cl),e(Za),e(xl),e(Va),e(zl),e(jl),e(Ea),e(_l),e(Il),e(ae)),e(M),d(j,s),d($,s),d(B,s),d(Z,s),d(E,s),d(W,s),d(N,s),d(O,s),d(K,s),d(es,s),d(is,s),d(os,s),d(hs,s),d(Ms,s),d(ws,s),d(Us,s),d(xs,s),d(_s,s),d(Rs,s),d(Bs,s),d(Ps,s),d(Es,s),d(Ds,s),d(Ss,s),d(Ns,s),d(Ys,s),d(Ks,s),d(aa,s),d(ta,s),d(na,s),d(pa,s),d(oa,s),d(ha,s),d(ga,s),d(Ma,s),d(fa,s),d(ba,s),d(Ja,s),d(Ta,s),d(xa,s),d(ja,s),d(Ia,s),d(ka,s),d(Ga,s),d(Xa,s),d(La,s)}}}const Zp='{"title":"Înțelegerea Avansată a Optimizării Relative a Politicii de Grup (GRPO) în DeepSeekMath","local":"înțelegerea-avansată-a-optimizării-relative-a-politicii-de-grup-grpo-în-deepseekmath","sections":[{"title":"Algoritmul GRPO","local":"algoritmul-grpo","sections":[{"title":"Pasul 1: Eșantionarea Grupului","local":"pasul-1-eșantionarea-grupului","sections":[{"title":"Exemplu:","local":"exemplu","sections":[],"depth":4}],"depth":3},{"title":"Pasul 2: Calculul Avantajului","local":"pasul-2-calculul-avantajului","sections":[{"title":"Distribuția Recompenselor:","local":"distribuția-recompenselor","sections":[],"depth":4},{"title":"Formula Valorii Avantajului:","local":"formula-valorii-avantajului","sections":[],"depth":4},{"title":"Exemplu:","local":"exemplu","sections":[],"depth":4},{"title":"Interpretarea:","local":"interpretarea","sections":[],"depth":4}],"depth":3},{"title":"Pasul 3: Actualizarea Politicii","local":"pasul-3-actualizarea-politicii","sections":[],"depth":3}],"depth":2},{"title":"Componentele Cheie ale Funcției Țintă","local":"componentele-cheie-ale-funcției-țintă","sections":[{"title":"1. Raportul de Probabilitate","local":"1-raportul-de-probabilitate","sections":[{"title":"Interpretarea:","local":"interpretarea","sections":[],"depth":4}],"depth":3},{"title":"2. Funcția de Tăiere","local":"2-funcția-de-tăiere","sections":[{"title":"Exemplu $\\space \\text{să presupunem}(\\epsilon = 0.2)$","local":"exemplu-space-textsă-presupunemepsilon--02","sections":[],"depth":4},{"title":"Interpretarea:","local":"interpretarea","sections":[],"depth":4}],"depth":3},{"title":"3. Divergența KL","local":"3-divergența-kl","sections":[{"title":"Interpretarea","local":"interpretarea","sections":[],"depth":4},{"title":"Definiția Matematică","local":"definiția-matematică","sections":[],"depth":4},{"title":"Rolul Parametrului $\\beta$","local":"rolul-parametrului-beta","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Exemplu Lucrat cu GRPO","local":"exemplu-lucrat-cu-grpo","sections":[{"title":"Problema Exemplu","local":"problema-exemplu","sections":[],"depth":3},{"title":"Pasul 1: Eșantionarea Grupului","local":"pasul-1-eșantionarea-grupului","sections":[],"depth":3},{"title":"Pasul 2: Calculul Avantajului","local":"pasul-2-calculul-avantajului","sections":[],"depth":3},{"title":"Pasul 3: Actualizarea Politicii","local":"pasul-3-actualizarea-politicii","sections":[],"depth":3}],"depth":2},{"title":"Exemplu de Implementare","local":"exemplu-de-implementare","sections":[{"title":"1. Încărcarea Modelului și Generarea Răspunsurilor","local":"1-încărcarea-modelului-și-generarea-răspunsurilor","sections":[],"depth":3},{"title":"2. Calcularea Recompenselor","local":"2-calcularea-recompenselor","sections":[],"depth":3},{"title":"3. Actualizarea Politicii","local":"3-actualizarea-politicii","sections":[],"depth":3}],"depth":2},{"title":"Rezumat și Pași Următori","local":"rezumat-și-pași-următori","sections":[],"depth":2},{"title":"Referințe","local":"referințe","sections":[],"depth":2}],"depth":1}';function Vp(te){return _p(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wp extends Ap{constructor(M){super(),kp(this,M,Vp,Pp,jp,{})}}export{Wp as component}; | |
Xet Storage Details
- Size:
- 106 kB
- Xet hash:
- c00a3cb5fec206125f7f9e1d6a657fd970c4c3fec6d2715719daf110588be0cc
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.