course/pr_1213/es/_app/immutable/nodes/31.a5ca85f0.js

import{s as He,n as Fe,o as qe}from"../chunks/scheduler.505acc25.js";import{S as Se,i as Ne,e as p,s as n,c as r,r as Rt,H as Ns,h as Ye,a as m,d as t,b as l,f as Hs,g as c,j as i,u as Pt,A as Ys,k as Ft,l as Ds,m as e,n as o,t as h,o as g,p as d}from"../chunks/index.e22abd30.js";import{C as De,H as u,E as Oe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.a144e953.js";import{C as Os}from"../chunks/CodeBlock.f6688f67.js";function Ke(qt){let y,Ks,qs,sa,f,aa,b,ta,v,St='<p>Esta sección profundiza en los detalles técnicos y matemáticos de GRPO. Fue escrita por <a href="https://huggingface.com/shirinyamani" target="_blank">Shirin Yamani</a>.</p>',ea,M,Nt="Profundicemos en nuestra comprensión de GRPO para poder mejorar el proceso de entrenamiento de nuestro modelo.",na,w,Yt="GRPO evalúa directamente las respuestas generadas por el modelo comparándolas dentro de grupos de generaciones para optimizar el modelo de política, en lugar de entrenar un modelo de valor separado. Este enfoque reduce de forma importante el coste computacional.",la,$,Dt="GRPO puede aplicarse a cualquier tarea verificable en la que pueda determinarse si la respuesta es correcta. Por ejemplo, en razonamiento matemático, esa corrección puede verificarse comparando con la verdad de referencia.",pa,T,Ot="Antes de entrar en los detalles técnicos, visualicemos cómo funciona GRPO a alto nivel:",ma,x,Kt='<img src="https://huggingface.co/reasoning-course/images/resolve/main/grpo/16.png" alt="deep"/>',ia,j,se="Ahora que tenemos una visión general, desglosaremos GRPO paso a paso.",ra,J,ca,_,ae="La innovación central de GRPO está en su forma de evaluar y aprender de múltiples respuestas generadas simultáneamente. En vez de depender de un reward model separado, compara salidas dentro del mismo grupo para decidir cuáles deben reforzarse.",oa,z,ha,C,te="El primer paso es generar varias respuestas posibles para cada pregunta. Esto crea un conjunto diverso de salidas que pueden compararse entre sí.",ga,U,ee="Para cada pregunta ( q ), el modelo genera ( G ) salidas a partir de la política entrenada, donde ( G ) es el tamaño del grupo.",da,k,ua,X,ne="Considera este problema aritmético sencillo:",ya,B,le="<strong>Pregunta</strong>",va,I,pe="( q ): ( \\text{Calculate}\\space2 + 2 \\times 6 )",fa,G,me="<strong>Salidas</strong>",ba,R,ie="( G = 8 ): ( {o_1:14 \\text{ (correct)}, o_2:16 \\text{ (wrong)}, o_3:10 \\text{ (wrong)}, \\ldots, o_8:14 \\text{ (correct)}} )",Ma,P,re="Observa que algunas respuestas son correctas y otras no. Esa diversidad es crucial para el siguiente paso.",wa,E,$a,Z,ce="Una vez que tenemos varias respuestas, necesitamos una forma de determinar cuáles son mejores que las demás.",Ta,V,xa,L,oe="Primero asignamos una puntuación de recompensa a cada respuesta generada. En este ejemplo usaremos un reward model, aunque también podría ser cualquier función que devuelva recompensas.",ja,A,he="Asignamos una recompensa ( r_i ) a cada respuesta y luego calculamos el siguiente valor de ventaja.",Ja,W,_a,Q,Et,za,Ve='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo>=</mo><mfrac><mrow><msub><mi>r</mi><mi>i</mi></msub><mo>−</mo><mtext>mean</mtext><mo stretchy="false">(</mo><mo stretchy="false">{</mo><msub><mi>r</mi><mn>1</mn></msub><mo separator="true">,</mo><msub><mi>r</mi><mn>2</mn></msub><mo separator="true">,</mo><mo>…</mo><mo separator="true">,</mo><msub><mi>r</mi><mi>G</mi></msub><mo stretchy="false">}</mo><mo stretchy="false">)</mo></mrow><mrow><mtext>std</mtext><mo stretchy="false">(</mo><mo stretchy="false">{</mo><msub><mi>r</mi><mn>1</mn></msub><mo separator="true">,</mo><msub><mi>r</mi><mn>2</mn></msub><mo separator="true">,</mo><mo>…</mo><mo separator="true">,</mo><msub><mi>r</mi><mi>G</mi></msub><mo stretchy="false">}</mo><mo stretchy="false">)</mo></mrow></mfrac></mrow><annotation encoding="application/x-tex">A_i = \\frac{r_i - \\text{mean}(\\{r_1, r_2, \\ldots, r_G\\})}{\\text{std}(\\{r_1, r_2, \\ldots, r_G\\})}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.363em;vertical-align:-0.936em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord text"><span class="mord">std</span></span><span class="mopen">({</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner">…</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">G</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">})</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord text"><span class="mord">mean</span></span><span class="mopen">({</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner">…</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">G</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">})</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.936em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>',Ca,H,Ua,F,ge="Si tenemos 8 respuestas y 4 son correctas:",ka,q,de="<thead><tr><th>Métrica</th> <th>Valor</th></tr></thead> <tbody><tr><td>Promedio del grupo</td> <td>( mean(r_i) = 0.5 )</td></tr> <tr><td>Desviación estándar</td> <td>( std(r_i) = 0.53 )</td></tr> <tr><td>Valor de ventaja para respuesta correcta</td> <td>( A_i = \\frac{1 - 0.5}{0.53}= 0.94 )</td></tr> <tr><td>Valor de ventaja para respuesta incorrecta</td> <td>( A_i = \\frac{0 - 0.5}{0.53}= -0.94 )</td></tr></tbody>",Xa,S,Ba,N,ue="Esta estandarización permite que el modelo evalúe el rendimiento relativo de cada respuesta. Si ( A_i > 0 ), la respuesta está por encima del promedio del grupo. Si ( A_i < 0 ), está por debajo.",Ia,Y,Ga,D,ye="El último paso consiste en usar esos valores de ventaja para actualizar el modelo y hacer más probable que genere buenas respuestas en el futuro.",Ra,O,Zt,Pa,Le='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>J</mi><mrow><mi>G</mi><mi>R</mi><mi>P</mi><mi>O</mi></mrow></msub><mo stretchy="false">(</mo><mi>θ</mi><mo stretchy="false">)</mo><mo>=</mo><mrow><mo fence="true">[</mo><mfrac><mn>1</mn><mi>G</mi></mfrac><munderover><mo>∑</mo><mrow><mi>i</mi><mo>=</mo><mn>1</mn></mrow><mi>G</mi></munderover><mi>min</mi><mo>⁡</mo><mrow><mo fence="true">(</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><msub><mi>A</mi><mi>i</mi></msub><mo separator="true">,</mo><mtext>clip</mtext><mrow><mo fence="true">(</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><mo separator="true">,</mo><mn>1</mn><mo>−</mo><mi>ϵ</mi><mo separator="true">,</mo><mn>1</mn><mo>+</mo><mi>ϵ</mi><mo fence="true">)</mo></mrow><msub><mi>A</mi><mi>i</mi></msub><mo fence="true">)</mo></mrow><mo fence="true">]</mo></mrow><mo>−</mo><mi>β</mi><msub><mi>D</mi><mrow><mi>K</mi><mi>L</mi></mrow></msub><mo stretchy="false">(</mo><msub><mi>π</mi><mi>θ</mi></msub><mi mathvariant="normal">∥</mi><mi mathvariant="normal">∥</mi><msub><mi>π</mi><mrow><mi>r</mi><mi>e</mi><mi>f</mi></mrow></msub><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">J_{GRPO}(\\theta) = \\left[\\frac{1}{G} \\sum_{i=1}^{G} \\min \\left( \\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)} A_i, \\text{clip}\\left( \\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)}, 1 - \\epsilon, 1 + \\epsilon \\right) A_i \\right)\\right] - \\beta D_{KL}(\\pi_{\\theta} \\|\\| \\pi_{ref})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.09618em;">J</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0962em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">GRPO</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.02778em;">θ</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:3.106em;vertical-align:-1.2777em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size4">[</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">G</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.8283em;"><span style="top:-1.8723em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">i</span><span class="mrel mtight">=</span><span class="mord mtight">1</span></span></span></span><span style="top:-3.05em;"><span class="pstrut" style="height:3.05em;"></span><span><span class="mop op-symbol large-op">∑</span></span></span><span style="top:-4.3em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">G</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:1.2777em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop">min</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0278em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2559em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.9419em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord text"><span class="mord">clip</span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0278em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2559em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.9419em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal">ϵ</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal">ϵ</span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size4">]</span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.0361em;vertical-align:-0.2861em;"></span><span class="mord mathnormal" style="margin-right:0.05278em;">β</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">D</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.07153em;">K</span><span class="mord mathnormal mtight">L</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∥∥</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">re</span><span class="mord mathnormal mtight" style="margin-right:0.10764em;">f</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span><span class="mclose">)</span></span></span></span></span>',Ea,K,Za,ss,Va,as,ve="El cociente de probabilidad es:",La,ts,fe="( \\frac{\\pi<em>{\\theta}(o_i|q)}{\\pi</em>{\\theta_{old}}(o_i|q)} )",Aa,es,be="Compara cuánto cambia la probabilidad asignada por el nuevo modelo respecto del modelo anterior.",Wa,ns,Qa,ls,Me="La función de recorte es:",Ha,ps,we="( \\text{clip}\\left( \\frac{\\pi<em>{\\theta}(o_i|q)}{\\pi</em>{\\theta_{old}}(o_i|q)}, 1 - \\epsilon, 1 + \\epsilon\\right) )",Fa,ms,$e="Limita el cociente al intervalo ( [1 - \\epsilon, 1 + \\epsilon] ) para evitar cambios demasiado bruscos.",qa,is,Sa,rs,Te="<li>Si el cociente es 1.8, se recorta a 1.2.</li> <li>Si el cociente es 0.4, se recorta a 0.8.</li>",Na,cs,Ya,os,xe="El término de divergencia KL es:",Da,hs,je="( \\beta D<em>{KL}(\\pi</em>{\\theta} || \\pi_{ref}) )",Oa,gs,Je="La penalización KL evita que el modelo se desvíe demasiado de su comportamiento original durante la optimización.",Ka,ds,Vt,st,Ae='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>D</mi><mrow><mi>K</mi><mi>L</mi></mrow></msub><mo stretchy="false">(</mo><mi>P</mi><mi mathvariant="normal">∥</mi><mi mathvariant="normal">∥</mi><mi>Q</mi><mo stretchy="false">)</mo><mo>=</mo><munder><mo>∑</mo><mrow><mi>x</mi><mo>∈</mo><mi>X</mi></mrow></munder><mi>P</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo><mi>log</mi><mo>⁡</mo><mfrac><mrow><mi>P</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo></mrow><mrow><mi>Q</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo></mrow></mfrac></mrow><annotation encoding="application/x-tex">D_{KL}(P \\|\\| Q) = \\sum_{x \\in X} P(x) \\log \\frac{P(x)}{Q(x)}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">D</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.07153em;">K</span><span class="mord mathnormal mtight">L</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mord">∥∥</span><span class="mord mathnormal">Q</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.7487em;vertical-align:-1.3217em;"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.05em;"><span style="top:-1.8557em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">x</span><span class="mrel mtight">∈</span><span class="mord mathnormal mtight" style="margin-right:0.07847em;">X</span></span></span></span><span style="top:-3.05em;"><span class="pstrut" style="height:3.05em;"></span><span><span class="mop op-symbol large-op">∑</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:1.3217em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop">lo<span style="margin-right:0.01389em;">g</span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">Q</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.936em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>',at,us,_e="El coeficiente ( \\beta ) controla con qué fuerza imponemos esa restricción.",tt,ys,et,vs,ze="Para afianzar la intuición, recorramos un ejemplo completo.",nt,fs,lt,Fs,pt,We='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mtext>Q: Calculate </mtext><mn>2</mn><mo>+</mo><mn>2</mn><mo>×</mo><mn>6</mn></mrow><annotation encoding="application/x-tex">\\text{Q: Calculate}\\space2 + 2 \\times 6</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord text"><span class="mord">Q: Calculate</span></span><span class="mspace"> </span><span class="mord">2</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.7278em;vertical-align:-0.0833em;"></span><span class="mord">2</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">6</span></span></span></span></span>',mt,bs,it,Ms,Ce="Generamos ( G = 8 ) respuestas, de las cuales 4 son correctas y 4 incorrectas.",rt,ws,ct,$s,Ue="<thead><tr><th>Estadística</th> <th>Valor</th></tr></thead> <tbody><tr><td>Promedio del grupo</td> <td>( mean(r_i) = 0.5 )</td></tr> <tr><td>Desviación estándar</td> <td>( std(r_i) = 0.53 )</td></tr> <tr><td>Ventaja de respuesta correcta</td> <td>( A_i = \\frac{1 - 0.5}{0.53}= 0.94 )</td></tr> <tr><td>Ventaja de respuesta incorrecta</td> <td>( A_i = \\frac{0 - 0.5}{0.53}= -0.94 )</td></tr></tbody>",ot,Ts,ht,xs,Lt,gt,Qe='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mtext>Ratio</mtext><mo>:</mo><mfrac><mn>0.7</mn><mn>0.5</mn></mfrac><mo>=</mo><mn>1.4</mn><mo>→</mo><mtext>after Clip </mtext><mn>1.2</mn><mtext> </mtext><mo stretchy="false">(</mo><mi>ϵ</mi><mo>=</mo><mn>0.2</mn><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">\\text{Ratio}: \\frac{0.7}{0.5} = 1.4 \\rightarrow \\text{after Clip}\\space1.2 \\space (\\epsilon = 0.2)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord text"><span class="mord">Ratio</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">0.5</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">0.7</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">1.4</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">→</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord text"><span class="mord">after Clip</span></span><span class="mspace"> </span><span class="mord">1.2</span><span class="mspace"> </span><span class="mopen">(</span><span class="mord mathnormal">ϵ</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">0.2</span><span class="mclose">)</span></span></span></span></span>',dt,js,ke="Entonces el modelo tenderá a reforzar esa salida correcta, mientras la divergencia KL limita cuánto se aleja de la política de referencia.",ut,Js,yt,_s,vt,zs,ft,Cs,Xe="La generación inicial puede producir algo así:",bt,Us,Mt,ks,wt,Xs,$t,Bs,Tt,Is,xt,Gs,Be="<code>per_token_kl</code> puede calcularse así:",jt,Rs,Jt,Ps,Ie='Puedes encontrar un ejemplo completo <a href="./basic_example.py">aquí</a>. El equipo de TRL también implementó GRPO; revisa <a href="https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_trainer.py" rel="nofollow">TRL/GRPO_trainer</a> para más detalles.',_t,Es,zt,Zs,Ge="Ya has visto los elementos esenciales de Group Relative Policy Optimization:",Ct,Vs,Re="<li>GRPO compara múltiples salidas dentro de un grupo.</li> <li>El cálculo de la ventaja estandariza las recompensas.</li> <li>La actualización de la política usa clipping y una penalización KL para mantener estabilidad.</li>",Ut,Ls,Pe="Este enfoque es especialmente útil en tareas verificables, como razonamiento matemático.",kt,As,Xt,Ws,Ee='<li><a href="https://github.com/natolambert/rlhf-book" rel="nofollow">RLHF Book by Nathan Lambert</a></li> <li><a href="https://huggingface.co/papers/2412.19437" rel="nofollow">DeepSeek-V3 Technical Report</a></li> <li><a href="https://huggingface.co/papers/2402.03300" rel="nofollow">DeepSeekMath</a></li>',Bt,Qs,It,Ss,Gt;return f=new De({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),b=new u({props:{title:"Comprensión avanzada de Group Relative Policy Optimization (GRPO) en DeepSeekMath",local:"comprensión-avanzada-de-group-relative-policy-optimization-grpo-en-deepseekmath",headingTag:"h1"}}),J=new u({props:{title:"El algoritmo GRPO",local:"el-algoritmo-grpo",headingTag:"h2"}}),z=new u({props:{title:"Paso 1: muestreo de grupos",local:"paso-1-muestreo-de-grupos",headingTag:"h3"}}),k=new u({props:{title:"Ejemplo",local:"ejemplo",headingTag:"h4"}}),E=new u({props:{title:"Paso 2: cálculo de la ventaja",local:"paso-2-cálculo-de-la-ventaja",headingTag:"h3"}}),V=new u({props:{title:"Distribución de recompensas",local:"distribución-de-recompensas",headingTag:"h4"}}),W=new u({props:{title:"Fórmula de la ventaja",local:"fórmula-de-la-ventaja",headingTag:"h4"}}),H=new u({props:{title:"Ejemplo",local:"ejemplo",headingTag:"h4"}}),S=new u({props:{title:"Interpretación",local:"interpretación",headingTag:"h4"}}),Y=new u({props:{title:"Paso 3: actualización de la política",local:"paso-3-actualización-de-la-política",headingTag:"h3"}}),K=new u({props:{title:"Componentes clave de la función objetivo",local:"componentes-clave-de-la-función-objetivo",headingTag:"h2"}}),ss=new u({props:{title:"1. Cociente de probabilidad",local:"1-cociente-de-probabilidad",headingTag:"h3"}}),ns=new u({props:{title:"2. Función de recorte",local:"2-función-de-recorte",headingTag:"h3"}}),is=new u({props:{title:"Ejemplo con ( \\epsilon = 0.2 )",local:"ejemplo-con--epsilon--02-",headingTag:"h4"}}),cs=new u({props:{title:"3. Divergencia KL",local:"3-divergencia-kl",headingTag:"h3"}}),ys=new u({props:{title:"Ejemplo resuelto con GRPO",local:"ejemplo-resuelto-con-grpo",headingTag:"h2"}}),fs=new u({props:{title:"Problema de ejemplo",local:"problema-de-ejemplo",headingTag:"h3"}}),bs=new u({props:{title:"Paso 1: muestreo de grupos",local:"paso-1-muestreo-de-grupos",headingTag:"h3"}}),ws=new u({props:{title:"Paso 2: cálculo de la ventaja",local:"paso-2-cálculo-de-la-ventaja",headingTag:"h3"}}),Ts=new u({props:{title:"Paso 3: actualización de la política",local:"paso-3-actualización-de-la-política",headingTag:"h3"}}),Js=new u({props:{title:"Ejemplo de implementación",local:"ejemplo-de-implementación",headingTag:"h2"}}),_s=new u({props:{title:"1. Cargar el modelo y generar respuestas",local:"1-cargar-el-modelo-y-generar-respuestas",headingTag:"h3"}}),zs=new Os({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaC5ubi5mdW5jdGlvbmFsJTIwYXMlMjBGJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQSUyMyUyMExvYWQlMjB0aGUlMjBtb2RlbCUyMGFuZCUyMHRva2VuaXplciUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJRd2VuJTJGUXdlbjItTWF0aC0xLjVCJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9uYW1lKSUwQW1vZGVsLmV2YWwoKSUwQSUwQSUyMyUyME1vdmUlMjBtb2RlbCUyMHRvJTIwR1BVJTIwaWYlMjBhdmFpbGFibGUlMEFkZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3VkYSUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIpJTBBbW9kZWwudG8oZGV2aWNlKSUwQSUwQSUyMyUyMElucHV0JTIwcHJvbXB0JTBBcHJvbXB0JTIwJTNEJTIwJTIyU29sdmUlMjB5JTIwJTNEJTIwMnglMjAlMkIlMjAxJTIwZm9yJTIweCUyMCUzRCUyMDIlMkMlMjB5JTIwJTNEJTIwJTIyJTIwJTIwJTIzJTIwQ29ycmVjdCUyMGFuc3dlciUzQSUyMDUlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUyQyUyMHBhZGRpbmclM0RUcnVlKSUwQWlucHV0X2lkcyUyMCUzRCUyMGlucHV0cyU1QiUyMmlucHV0X2lkcyUyMiU1RC50byhkZXZpY2UpJTBBYXR0ZW50aW9uX21hc2slMjAlM0QlMjBpbnB1dHMlNUIlMjJhdHRlbnRpb25fbWFzayUyMiU1RC50byhkZXZpY2UpJTBBJTBBJTIzJTIwU3RlcCUyMDElM0ElMjBHZW5lcmF0ZSUyMDglMjByZXNwb25zZXMlMjAoQiUyMCUzRCUyMDIlMjBncm91cHMlMkMlMjBHJTIwJTNEJTIwNCUyMHJlc3BvbnNlcyUyMHBlciUyMGdyb3VwKSUwQWJhdGNoX3NpemUlMkMlMjBudW1fZ2VuZXJhdGlvbnMlMjAlM0QlMjAyJTJDJTIwNCUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSglMEElMjAlMjAlMjAlMjBpbnB1dF9pZHMlM0RpbnB1dF9pZHMlMkMlMEElMjAlMjAlMjAlMjBhdHRlbnRpb25fbWFzayUzRGF0dGVudGlvbl9tYXNrJTJDJTBBJTIwJTIwJTIwJTIwbWF4X25ld190b2tlbnMlM0QxJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3JldHVybl9zZXF1ZW5jZXMlM0RiYXRjaF9zaXplJTIwKiUyMG51bV9nZW5lcmF0aW9ucyUyQyUwQSUyMCUyMCUyMCUyMGRvX3NhbXBsZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjB0b3BfayUzRDEwJTJDJTBBJTIwJTIwJTIwJTIwdGVtcGVyYXR1cmUlM0QwLjclMkMlMEElMjAlMjAlMjAlMjBwYWRfdG9rZW5faWQlM0R0b2tlbml6ZXIuZW9zX3Rva2VuX2lkJTJDJTBBJTIwJTIwJTIwJTIwcmV0dXJuX2RpY3RfaW5fZ2VuZXJhdGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwb3V0cHV0X3Njb3JlcyUzRFRydWUlMkMlMEEp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.nn.functional <span class="hljs-keyword">as</span> F
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

<span class="hljs-comment"># Load the model and tokenizer</span>
model_name = <span class="hljs-string">&quot;Qwen/Qwen2-Math-1.5B&quot;</span>
model = AutoModelForCausalLM.from_pretrained(model_name)
tokenizer = AutoTokenizer.from_pretrained(model_name)
model.<span class="hljs-built_in">eval</span>()

<span class="hljs-comment"># Move model to GPU if available</span>
device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>)
model.to(device)

<span class="hljs-comment"># Input prompt</span>
prompt = <span class="hljs-string">&quot;Solve y = 2x + 1 for x = 2, y = &quot;</span>  <span class="hljs-comment"># Correct answer: 5</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, padding=<span class="hljs-literal">True</span>)
input_ids = inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].to(device)
attention_mask = inputs[<span class="hljs-string">&quot;attention_mask&quot;</span>].to(device)

<span class="hljs-comment"># Step 1: Generate 8 responses (B = 2 groups, G = 4 responses per group)</span>
batch_size, num_generations = <span class="hljs-number">2</span>, <span class="hljs-number">4</span>
outputs = model.generate(
    input_ids=input_ids,
    attention_mask=attention_mask,
    max_new_tokens=<span class="hljs-number">1</span>,
    num_return_sequences=batch_size * num_generations,
    do_sample=<span class="hljs-literal">True</span>,
    top_k=<span class="hljs-number">10</span>,
    temperature=<span class="hljs-number">0.7</span>,
    pad_token_id=tokenizer.eos_token_id,
    return_dict_in_generate=<span class="hljs-literal">True</span>,
    output_scores=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),Us=new Os({props:{code:"T3V0cHV0JTIwMSUzQSUyMDUuMCUwQU91dHB1dCUyMDIlM0ElMjA2LjAlMEFPdXRwdXQlMjAzJTNBJTIwNy4wJTBBT3V0cHV0JTIwNCUzQSUyMDUuMCUwQU91dHB1dCUyMDUlM0ElMjAxMC4wJTBBT3V0cHV0JTIwNiUzQSUyMDIuMCUwQU91dHB1dCUyMDclM0ElMjA1LjAlMEFPdXRwdXQlMjA4JTNBJTIwNS4w",highlighted:`Output 1: 5.0
Output 2: 6.0
Output 3: 7.0
Output 4: 5.0
Output 5: 10.0
Output 6: 2.0
Output 7: 5.0
Output 8: 5.0`,wrap:!1}}),ks=new u({props:{title:"2. Calcular recompensas",local:"2-calcular-recompensas",headingTag:"h3"}}),Xs=new Os({props:{code:"cmV3YXJkXzElMjAlM0QlMjAlNUIxJTJDJTIwMCUyQyUyMDAlMkMlMjAxJTVEJTBBcmV3YXJkXzIlMjAlM0QlMjAlNUIwJTJDJTIwMCUyQyUyMDElMkMlMjAxJTVEJTBBJTBBcmV3YXJkcyUyMCUzRCUyMHRvcmNoLnRlbnNvciglNUIxJTJDJTIwMCUyQyUyMDAlMkMlMjAxJTJDJTIwMCUyQyUyMDAlMkMlMjAxJTJDJTIwMSU1RCUyQyUyMGR0eXBlJTNEdG9yY2guZmxvYXQzMiklMEFudW1fZ2VuZXJhdGlvbnMlMjAlM0QlMjA0JTBBcmV3YXJkc19ncm91cGVkJTIwJTNEJTIwcmV3YXJkcy52aWV3KC0xJTJDJTIwbnVtX2dlbmVyYXRpb25zKSUwQW1lYW5fZ3JvdXBlZF9yZXdhcmRzJTIwJTNEJTIwcmV3YXJkc19ncm91cGVkLm1lYW4oZGltJTNEMSklMEFzdGRfZ3JvdXBlZF9yZXdhcmRzJTIwJTNEJTIwcmV3YXJkc19ncm91cGVkLnN0ZChkaW0lM0QxKSUwQW1lYW5fZ3JvdXBlZF9yZXdhcmRzJTIwJTNEJTIwbWVhbl9ncm91cGVkX3Jld2FyZHMucmVwZWF0X2ludGVybGVhdmUobnVtX2dlbmVyYXRpb25zJTJDJTIwZGltJTNEMCklMEFzdGRfZ3JvdXBlZF9yZXdhcmRzJTIwJTNEJTIwc3RkX2dyb3VwZWRfcmV3YXJkcy5yZXBlYXRfaW50ZXJsZWF2ZShudW1fZ2VuZXJhdGlvbnMlMkMlMjBkaW0lM0QwKSUwQSUwQWFkdmFudGFnZXMlMjAlM0QlMjAocmV3YXJkcyUyMC0lMjBtZWFuX2dyb3VwZWRfcmV3YXJkcyklMjAlMkYlMjAoc3RkX2dyb3VwZWRfcmV3YXJkcyUyMCUyQiUyMDFlLTgpJTBBYWR2YW50YWdlcyUyMCUzRCUyMGFkdmFudGFnZXMudW5zcXVlZXplKDEp",highlighted:`reward_1 = [<span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
reward_2 = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]

rewards = torch.tensor([<span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], dtype=torch.float32)
num_generations = <span class="hljs-number">4</span>
rewards_grouped = rewards.view(-<span class="hljs-number">1</span>, num_generations)
mean_grouped_rewards = rewards_grouped.mean(dim=<span class="hljs-number">1</span>)
std_grouped_rewards = rewards_grouped.std(dim=<span class="hljs-number">1</span>)
mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(num_generations, dim=<span class="hljs-number">0</span>)
std_grouped_rewards = std_grouped_rewards.repeat_interleave(num_generations, dim=<span class="hljs-number">0</span>)

advantages = (rewards - mean_grouped_rewards) / (std_grouped_rewards + <span class="hljs-number">1e-8</span>)
advantages = advantages.unsqueeze(<span class="hljs-number">1</span>)`,wrap:!1}}),Bs=new u({props:{title:"3. Actualizar la política",local:"3-actualizar-la-política",headingTag:"h3"}}),Is=new Os({props:{code:"cmF0aW8lMjAlM0QlMjB0b3JjaC5leHAobmV3X3Blcl90b2tlbl9sb2dwcyUyMC0lMjBwZXJfdG9rZW5fbG9ncHMpJTBBJTBBZXBzJTIwJTNEJTIwc2VsZi5jbGlwcmFuZ2UlMEFwZ19sb3NzZXMxJTIwJTNEJTIwLWFkdmFudGFnZXMlMjAqJTIwcmF0aW8lMEFwZ19sb3NzZXMyJTIwJTNEJTIwLWFkdmFudGFnZXMlMjAqJTIwdG9yY2guY2xhbXAocmF0aW8lMkMlMjAxLjAlMjAtJTIwZXBzJTJDJTIwMS4wJTIwJTJCJTIwZXBzKSUwQXBnX2xvc3NfbWF4JTIwJTNEJTIwdG9yY2gubWF4KHBnX2xvc3NlczElMkMlMjBwZ19sb3NzZXMyKSUwQXBlcl90b2tlbl9sb3NzJTIwJTNEJTIwcGdfbG9zc19tYXglMjAlMkIlMjBzZWxmLmJldGElMjAqJTIwcGVyX3Rva2VuX2ts",highlighted:`ratio = torch.exp(new_per_token_logps - per_token_logps)

eps = self.cliprange
pg_losses1 = -advantages * ratio
pg_losses2 = -advantages * torch.clamp(ratio, <span class="hljs-number">1.0</span> - eps, <span class="hljs-number">1.0</span> + eps)
pg_loss_max = torch.<span class="hljs-built_in">max</span>(pg_losses1, pg_losses2)
per_token_loss = pg_loss_max + self.beta * per_token_kl`,wrap:!1}}),Rs=new Os({props:{code:"cGVyX3Rva2VuX2tsJTIwJTNEJTIwRi5rbF9kaXYoJTBBJTIwJTIwJTIwJTIwRi5sb2dfc29mdG1heChuZXdfcGVyX3Rva2VuX2xvZ3BzJTJDJTIwZGltJTNELTEpJTJDJTBBJTIwJTIwJTIwJTIwRi5zb2Z0bWF4KHBlcl90b2tlbl9sb2dwcyUyQyUyMGRpbSUzRC0xKSUyQyUwQSUyMCUyMCUyMCUyMHJlZHVjdGlvbiUzRCUyMm5vbmUlMjIlMkMlMEEpLnN1bShkaW0lM0QtMSUyQyUyMGtlZXBkaW0lM0RUcnVlKQ==",highlighted:`per_token_kl = F.kl_div(
    F.log_softmax(new_per_token_logps, dim=-<span class="hljs-number">1</span>),
    F.softmax(per_token_logps, dim=-<span class="hljs-number">1</span>),
    reduction=<span class="hljs-string">&quot;none&quot;</span>,
).<span class="hljs-built_in">sum</span>(dim=-<span class="hljs-number">1</span>, keepdim=<span class="hljs-literal">True</span>)`,wrap:!1}}),Es=new u({props:{title:"Resumen y próximos pasos",local:"resumen-y-próximos-pasos",headingTag:"h2"}}),As=new u({props:{title:"Referencias",local:"referencias",headingTag:"h2"}}),Qs=new Oe({props:{source:"https://github.com/huggingface/course/blob/main/chapters/es/chapter12/3b.mdx"}}),{c(){y=p("meta"),Ks=n(),qs=p("p"),sa=n(),r(f.$$.fragment),aa=n(),r(b.$$.fragment),ta=n(),v=p("blockquote"),v.innerHTML=St,ea=n(),M=p("p"),M.textContent=Nt,na=n(),w=p("p"),w.textContent=Yt,la=n(),$=p("p"),$.textContent=Dt,pa=n(),T=p("p"),T.textContent=Ot,ma=n(),x=p("p"),x.innerHTML=Kt,ia=n(),j=p("p"),j.textContent=se,ra=n(),r(J.$$.fragment),ca=n(),_=p("p"),_.textContent=ae,oa=n(),r(z.$$.fragment),ha=n(),C=p("p"),C.textContent=te,ga=n(),U=p("p"),U.textContent=ee,da=n(),r(k.$$.fragment),ua=n(),X=p("p"),X.textContent=ne,ya=n(),B=p("p"),B.innerHTML=le,va=n(),I=p("p"),I.textContent=pe,fa=n(),G=p("p"),G.innerHTML=me,ba=n(),R=p("p"),R.textContent=ie,Ma=n(),P=p("p"),P.textContent=re,wa=n(),r(E.$$.fragment),$a=n(),Z=p("p"),Z.textContent=ce,Ta=n(),r(V.$$.fragment),xa=n(),L=p("p"),L.textContent=oe,ja=n(),A=p("p"),A.textContent=he,Ja=n(),r(W.$$.fragment),_a=n(),Q=p("p"),Et=Rt(`La idea clave de GRPO es que no necesitamos medidas absolutas de calidad; podemos comparar respuestas dentro del mismo grupo. Eso se hace mediante estandarización:
`),za=new Ns(!1),Ca=n(),r(H.$$.fragment),Ua=n(),F=p("p"),F.textContent=ge,ka=n(),q=p("table"),q.innerHTML=de,Xa=n(),r(S.$$.fragment),Ba=n(),N=p("p"),N.textContent=ue,Ia=n(),r(Y.$$.fragment),Ga=n(),D=p("p"),D.textContent=ye,Ra=n(),O=p("p"),Zt=Rt(`La función objetivo para actualizar la política es:
`),Pa=new Ns(!1),Ea=n(),r(K.$$.fragment),Za=n(),r(ss.$$.fragment),Va=n(),as=p("p"),as.textContent=ve,La=n(),ts=p("p"),ts.innerHTML=fe,Aa=n(),es=p("p"),es.textContent=be,Wa=n(),r(ns.$$.fragment),Qa=n(),ls=p("p"),ls.textContent=Me,Ha=n(),ps=p("p"),ps.innerHTML=we,Fa=n(),ms=p("p"),ms.textContent=$e,qa=n(),r(is.$$.fragment),Sa=n(),rs=p("ul"),rs.innerHTML=Te,Na=n(),r(cs.$$.fragment),Ya=n(),os=p("p"),os.textContent=xe,Da=n(),hs=p("p"),hs.innerHTML=je,Oa=n(),gs=p("p"),gs.textContent=Je,Ka=n(),ds=p("p"),Vt=Rt(`La distancia KL se define así:
`),st=new Ns(!1),at=n(),us=p("p"),us.textContent=_e,tt=n(),r(ys.$$.fragment),et=n(),vs=p("p"),vs.textContent=ze,nt=n(),r(fs.$$.fragment),lt=n(),Fs=p("p"),pt=new Ns(!1),mt=n(),r(bs.$$.fragment),it=n(),Ms=p("p"),Ms.textContent=Ce,rt=n(),r(ws.$$.fragment),ct=n(),$s=p("table"),$s.innerHTML=Ue,ot=n(),r(Ts.$$.fragment),ht=n(),xs=p("p"),Lt=Rt(`Suponiendo que la política anterior asigna probabilidad 0.5 a una respuesta correcta y la nueva política la sube a 0.7:
`),gt=new Ns(!1),dt=n(),js=p("p"),js.textContent=ke,ut=n(),r(Js.$$.fragment),yt=n(),r(_s.$$.fragment),vt=n(),r(zs.$$.fragment),ft=n(),Cs=p("p"),Cs.textContent=Xe,bt=n(),r(Us.$$.fragment),Mt=n(),r(ks.$$.fragment),wt=n(),r(Xs.$$.fragment),$t=n(),r(Bs.$$.fragment),Tt=n(),r(Is.$$.fragment),xt=n(),Gs=p("p"),Gs.innerHTML=Be,jt=n(),r(Rs.$$.fragment),Jt=n(),Ps=p("p"),Ps.innerHTML=Ie,_t=n(),r(Es.$$.fragment),zt=n(),Zs=p("p"),Zs.textContent=Ge,Ct=n(),Vs=p("ol"),Vs.innerHTML=Re,Ut=n(),Ls=p("p"),Ls.textContent=Pe,kt=n(),r(As.$$.fragment),Xt=n(),Ws=p("ol"),Ws.innerHTML=Ee,Bt=n(),r(Qs.$$.fragment),It=n(),Ss=p("p"),this.h()},l(s){const a=Ye("svelte-u9bgzb",document.head);y=m(a,"META",{name:!0,content:!0}),a.forEach(t),Ks=l(s),qs=m(s,"P",{}),Hs(qs).forEach(t),sa=l(s),c(f.$$.fragment,s),aa=l(s),c(b.$$.fragment,s),ta=l(s),v=m(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),i(v)!=="svelte-stw9pv"&&(v.innerHTML=St),ea=l(s),M=m(s,"P",{"data-svelte-h":!0}),i(M)!=="svelte-2htoe4"&&(M.textContent=Nt),na=l(s),w=m(s,"P",{"data-svelte-h":!0}),i(w)!=="svelte-s2i67m"&&(w.textContent=Yt),la=l(s),$=m(s,"P",{"data-svelte-h":!0}),i($)!=="svelte-hr3f3w"&&($.textContent=Dt),pa=l(s),T=m(s,"P",{"data-svelte-h":!0}),i(T)!=="svelte-bjpz9f"&&(T.textContent=Ot),ma=l(s),x=m(s,"P",{"data-svelte-h":!0}),i(x)!=="svelte-1fuvyr0"&&(x.innerHTML=Kt),ia=l(s),j=m(s,"P",{"data-svelte-h":!0}),i(j)!=="svelte-94jxle"&&(j.textContent=se),ra=l(s),c(J.$$.fragment,s),ca=l(s),_=m(s,"P",{"data-svelte-h":!0}),i(_)!=="svelte-a04wzn"&&(_.textContent=ae),oa=l(s),c(z.$$.fragment,s),ha=l(s),C=m(s,"P",{"data-svelte-h":!0}),i(C)!=="svelte-1xmqcj0"&&(C.textContent=te),ga=l(s),U=m(s,"P",{"data-svelte-h":!0}),i(U)!=="svelte-13bthcv"&&(U.textContent=ee),da=l(s),c(k.$$.fragment,s),ua=l(s),X=m(s,"P",{"data-svelte-h":!0}),i(X)!=="svelte-1qf8jw7"&&(X.textContent=ne),ya=l(s),B=m(s,"P",{"data-svelte-h":!0}),i(B)!=="svelte-quuyab"&&(B.innerHTML=le),va=l(s),I=m(s,"P",{"data-svelte-h":!0}),i(I)!=="svelte-1ntvdcm"&&(I.textContent=pe),fa=l(s),G=m(s,"P",{"data-svelte-h":!0}),i(G)!=="svelte-p0rke8"&&(G.innerHTML=me),ba=l(s),R=m(s,"P",{"data-svelte-h":!0}),i(R)!=="svelte-11dan6t"&&(R.textContent=ie),Ma=l(s),P=m(s,"P",{"data-svelte-h":!0}),i(P)!=="svelte-1jlujge"&&(P.textContent=re),wa=l(s),c(E.$$.fragment,s),$a=l(s),Z=m(s,"P",{"data-svelte-h":!0}),i(Z)!=="svelte-t9lz83"&&(Z.textContent=ce),Ta=l(s),c(V.$$.fragment,s),xa=l(s),L=m(s,"P",{"data-svelte-h":!0}),i(L)!=="svelte-w3u6p3"&&(L.textContent=oe),ja=l(s),A=m(s,"P",{"data-svelte-h":!0}),i(A)!=="svelte-g0dlgh"&&(A.textContent=he),Ja=l(s),c(W.$$.fragment,s),_a=l(s),Q=m(s,"P",{});var At=Hs(Q);Et=Pt(At,`La idea clave de GRPO es que no necesitamos medidas absolutas de calidad; podemos comparar respuestas dentro del mismo grupo. Eso se hace mediante estandarización:
`),za=Ys(At,!1),At.forEach(t),Ca=l(s),c(H.$$.fragment,s),Ua=l(s),F=m(s,"P",{"data-svelte-h":!0}),i(F)!=="svelte-1uzpmbp"&&(F.textContent=ge),ka=l(s),q=m(s,"TABLE",{"data-svelte-h":!0}),i(q)!=="svelte-1k60mzw"&&(q.innerHTML=de),Xa=l(s),c(S.$$.fragment,s),Ba=l(s),N=m(s,"P",{"data-svelte-h":!0}),i(N)!=="svelte-iqgcvn"&&(N.textContent=ue),Ia=l(s),c(Y.$$.fragment,s),Ga=l(s),D=m(s,"P",{"data-svelte-h":!0}),i(D)!=="svelte-ncf72x"&&(D.textContent=ye),Ra=l(s),O=m(s,"P",{});var Wt=Hs(O);Zt=Pt(Wt,`La función objetivo para actualizar la política es:
`),Pa=Ys(Wt,!1),Wt.forEach(t),Ea=l(s),c(K.$$.fragment,s),Za=l(s),c(ss.$$.fragment,s),Va=l(s),as=m(s,"P",{"data-svelte-h":!0}),i(as)!=="svelte-4wbl5n"&&(as.textContent=ve),La=l(s),ts=m(s,"P",{"data-svelte-h":!0}),i(ts)!=="svelte-sa8r2d"&&(ts.innerHTML=fe),Aa=l(s),es=m(s,"P",{"data-svelte-h":!0}),i(es)!=="svelte-jz6c4z"&&(es.textContent=be),Wa=l(s),c(ns.$$.fragment,s),Qa=l(s),ls=m(s,"P",{"data-svelte-h":!0}),i(ls)!=="svelte-tq9ewk"&&(ls.textContent=Me),Ha=l(s),ps=m(s,"P",{"data-svelte-h":!0}),i(ps)!=="svelte-3a0rm1"&&(ps.innerHTML=we),Fa=l(s),ms=m(s,"P",{"data-svelte-h":!0}),i(ms)!=="svelte-m3joqs"&&(ms.textContent=$e),qa=l(s),c(is.$$.fragment,s),Sa=l(s),rs=m(s,"UL",{"data-svelte-h":!0}),i(rs)!=="svelte-wda7fy"&&(rs.innerHTML=Te),Na=l(s),c(cs.$$.fragment,s),Ya=l(s),os=m(s,"P",{"data-svelte-h":!0}),i(os)!=="svelte-sa1ve0"&&(os.textContent=xe),Da=l(s),hs=m(s,"P",{"data-svelte-h":!0}),i(hs)!=="svelte-188a2pl"&&(hs.innerHTML=je),Oa=l(s),gs=m(s,"P",{"data-svelte-h":!0}),i(gs)!=="svelte-8ti9r4"&&(gs.textContent=Je),Ka=l(s),ds=m(s,"P",{});var Qt=Hs(ds);Vt=Pt(Qt,`La distancia KL se define así:
`),st=Ys(Qt,!1),Qt.forEach(t),at=l(s),us=m(s,"P",{"data-svelte-h":!0}),i(us)!=="svelte-wdzzhl"&&(us.textContent=_e),tt=l(s),c(ys.$$.fragment,s),et=l(s),vs=m(s,"P",{"data-svelte-h":!0}),i(vs)!=="svelte-1wfu3v6"&&(vs.textContent=ze),nt=l(s),c(fs.$$.fragment,s),lt=l(s),Fs=m(s,"P",{});var Ze=Hs(Fs);pt=Ys(Ze,!1),Ze.forEach(t),mt=l(s),c(bs.$$.fragment,s),it=l(s),Ms=m(s,"P",{"data-svelte-h":!0}),i(Ms)!=="svelte-1cd5kh5"&&(Ms.textContent=Ce),rt=l(s),c(ws.$$.fragment,s),ct=l(s),$s=m(s,"TABLE",{"data-svelte-h":!0}),i($s)!=="svelte-uzrfjx"&&($s.innerHTML=Ue),ot=l(s),c(Ts.$$.fragment,s),ht=l(s),xs=m(s,"P",{});var Ht=Hs(xs);Lt=Pt(Ht,`Suponiendo que la política anterior asigna probabilidad 0.5 a una respuesta correcta y la nueva política la sube a 0.7:
`),gt=Ys(Ht,!1),Ht.forEach(t),dt=l(s),js=m(s,"P",{"data-svelte-h":!0}),i(js)!=="svelte-s0qqhg"&&(js.textContent=ke),ut=l(s),c(Js.$$.fragment,s),yt=l(s),c(_s.$$.fragment,s),vt=l(s),c(zs.$$.fragment,s),ft=l(s),Cs=m(s,"P",{"data-svelte-h":!0}),i(Cs)!=="svelte-mejha2"&&(Cs.textContent=Xe),bt=l(s),c(Us.$$.fragment,s),Mt=l(s),c(ks.$$.fragment,s),wt=l(s),c(Xs.$$.fragment,s),$t=l(s),c(Bs.$$.fragment,s),Tt=l(s),c(Is.$$.fragment,s),xt=l(s),Gs=m(s,"P",{"data-svelte-h":!0}),i(Gs)!=="svelte-cfq8jt"&&(Gs.innerHTML=Be),jt=l(s),c(Rs.$$.fragment,s),Jt=l(s),Ps=m(s,"P",{"data-svelte-h":!0}),i(Ps)!=="svelte-tp1ic1"&&(Ps.innerHTML=Ie),_t=l(s),c(Es.$$.fragment,s),zt=l(s),Zs=m(s,"P",{"data-svelte-h":!0}),i(Zs)!=="svelte-pyhjtg"&&(Zs.textContent=Ge),Ct=l(s),Vs=m(s,"OL",{"data-svelte-h":!0}),i(Vs)!=="svelte-13vzzou"&&(Vs.innerHTML=Re),Ut=l(s),Ls=m(s,"P",{"data-svelte-h":!0}),i(Ls)!=="svelte-g0kyc3"&&(Ls.textContent=Pe),kt=l(s),c(As.$$.fragment,s),Xt=l(s),Ws=m(s,"OL",{"data-svelte-h":!0}),i(Ws)!=="svelte-18yys75"&&(Ws.innerHTML=Ee),Bt=l(s),c(Qs.$$.fragment,s),It=l(s),Ss=m(s,"P",{}),Hs(Ss).forEach(t),this.h()},h(){Ft(y,"name","hf:doc:metadata"),Ft(y,"content",sn),Ft(v,"class","tip"),za.a=null,Pa.a=null,st.a=null,pt.a=null,gt.a=null},m(s,a){Ds(document.head,y),e(s,Ks,a),e(s,qs,a),e(s,sa,a),o(f,s,a),e(s,aa,a),o(b,s,a),e(s,ta,a),e(s,v,a),e(s,ea,a),e(s,M,a),e(s,na,a),e(s,w,a),e(s,la,a),e(s,$,a),e(s,pa,a),e(s,T,a),e(s,ma,a),e(s,x,a),e(s,ia,a),e(s,j,a),e(s,ra,a),o(J,s,a),e(s,ca,a),e(s,_,a),e(s,oa,a),o(z,s,a),e(s,ha,a),e(s,C,a),e(s,ga,a),e(s,U,a),e(s,da,a),o(k,s,a),e(s,ua,a),e(s,X,a),e(s,ya,a),e(s,B,a),e(s,va,a),e(s,I,a),e(s,fa,a),e(s,G,a),e(s,ba,a),e(s,R,a),e(s,Ma,a),e(s,P,a),e(s,wa,a),o(E,s,a),e(s,$a,a),e(s,Z,a),e(s,Ta,a),o(V,s,a),e(s,xa,a),e(s,L,a),e(s,ja,a),e(s,A,a),e(s,Ja,a),o(W,s,a),e(s,_a,a),e(s,Q,a),Ds(Q,Et),za.m(Ve,Q),e(s,Ca,a),o(H,s,a),e(s,Ua,a),e(s,F,a),e(s,ka,a),e(s,q,a),e(s,Xa,a),o(S,s,a),e(s,Ba,a),e(s,N,a),e(s,Ia,a),o(Y,s,a),e(s,Ga,a),e(s,D,a),e(s,Ra,a),e(s,O,a),Ds(O,Zt),Pa.m(Le,O),e(s,Ea,a),o(K,s,a),e(s,Za,a),o(ss,s,a),e(s,Va,a),e(s,as,a),e(s,La,a),e(s,ts,a),e(s,Aa,a),e(s,es,a),e(s,Wa,a),o(ns,s,a),e(s,Qa,a),e(s,ls,a),e(s,Ha,a),e(s,ps,a),e(s,Fa,a),e(s,ms,a),e(s,qa,a),o(is,s,a),e(s,Sa,a),e(s,rs,a),e(s,Na,a),o(cs,s,a),e(s,Ya,a),e(s,os,a),e(s,Da,a),e(s,hs,a),e(s,Oa,a),e(s,gs,a),e(s,Ka,a),e(s,ds,a),Ds(ds,Vt),st.m(Ae,ds),e(s,at,a),e(s,us,a),e(s,tt,a),o(ys,s,a),e(s,et,a),e(s,vs,a),e(s,nt,a),o(fs,s,a),e(s,lt,a),e(s,Fs,a),pt.m(We,Fs),e(s,mt,a),o(bs,s,a),e(s,it,a),e(s,Ms,a),e(s,rt,a),o(ws,s,a),e(s,ct,a),e(s,$s,a),e(s,ot,a),o(Ts,s,a),e(s,ht,a),e(s,xs,a),Ds(xs,Lt),gt.m(Qe,xs),e(s,dt,a),e(s,js,a),e(s,ut,a),o(Js,s,a),e(s,yt,a),o(_s,s,a),e(s,vt,a),o(zs,s,a),e(s,ft,a),e(s,Cs,a),e(s,bt,a),o(Us,s,a),e(s,Mt,a),o(ks,s,a),e(s,wt,a),o(Xs,s,a),e(s,$t,a),o(Bs,s,a),e(s,Tt,a),o(Is,s,a),e(s,xt,a),e(s,Gs,a),e(s,jt,a),o(Rs,s,a),e(s,Jt,a),e(s,Ps,a),e(s,_t,a),o(Es,s,a),e(s,zt,a),e(s,Zs,a),e(s,Ct,a),e(s,Vs,a),e(s,Ut,a),e(s,Ls,a),e(s,kt,a),o(As,s,a),e(s,Xt,a),e(s,Ws,a),e(s,Bt,a),o(Qs,s,a),e(s,It,a),e(s,Ss,a),Gt=!0},p:Fe,i(s){Gt||(h(f.$$.fragment,s),h(b.$$.fragment,s),h(J.$$.fragment,s),h(z.$$.fragment,s),h(k.$$.fragment,s),h(E.$$.fragment,s),h(V.$$.fragment,s),h(W.$$.fragment,s),h(H.$$.fragment,s),h(S.$$.fragment,s),h(Y.$$.fragment,s),h(K.$$.fragment,s),h(ss.$$.fragment,s),h(ns.$$.fragment,s),h(is.$$.fragment,s),h(cs.$$.fragment,s),h(ys.$$.fragment,s),h(fs.$$.fragment,s),h(bs.$$.fragment,s),h(ws.$$.fragment,s),h(Ts.$$.fragment,s),h(Js.$$.fragment,s),h(_s.$$.fragment,s),h(zs.$$.fragment,s),h(Us.$$.fragment,s),h(ks.$$.fragment,s),h(Xs.$$.fragment,s),h(Bs.$$.fragment,s),h(Is.$$.fragment,s),h(Rs.$$.fragment,s),h(Es.$$.fragment,s),h(As.$$.fragment,s),h(Qs.$$.fragment,s),Gt=!0)},o(s){g(f.$$.fragment,s),g(b.$$.fragment,s),g(J.$$.fragment,s),g(z.$$.fragment,s),g(k.$$.fragment,s),g(E.$$.fragment,s),g(V.$$.fragment,s),g(W.$$.fragment,s),g(H.$$.fragment,s),g(S.$$.fragment,s),g(Y.$$.fragment,s),g(K.$$.fragment,s),g(ss.$$.fragment,s),g(ns.$$.fragment,s),g(is.$$.fragment,s),g(cs.$$.fragment,s),g(ys.$$.fragment,s),g(fs.$$.fragment,s),g(bs.$$.fragment,s),g(ws.$$.fragment,s),g(Ts.$$.fragment,s),g(Js.$$.fragment,s),g(_s.$$.fragment,s),g(zs.$$.fragment,s),g(Us.$$.fragment,s),g(ks.$$.fragment,s),g(Xs.$$.fragment,s),g(Bs.$$.fragment,s),g(Is.$$.fragment,s),g(Rs.$$.fragment,s),g(Es.$$.fragment,s),g(As.$$.fragment,s),g(Qs.$$.fragment,s),Gt=!1},d(s){s&&(t(Ks),t(qs),t(sa),t(aa),t(ta),t(v),t(ea),t(M),t(na),t(w),t(la),t($),t(pa),t(T),t(ma),t(x),t(ia),t(j),t(ra),t(ca),t(_),t(oa),t(ha),t(C),t(ga),t(U),t(da),t(ua),t(X),t(ya),t(B),t(va),t(I),t(fa),t(G),t(ba),t(R),t(Ma),t(P),t(wa),t($a),t(Z),t(Ta),t(xa),t(L),t(ja),t(A),t(Ja),t(_a),t(Q),t(Ca),t(Ua),t(F),t(ka),t(q),t(Xa),t(Ba),t(N),t(Ia),t(Ga),t(D),t(Ra),t(O),t(Ea),t(Za),t(Va),t(as),t(La),t(ts),t(Aa),t(es),t(Wa),t(Qa),t(ls),t(Ha),t(ps),t(Fa),t(ms),t(qa),t(Sa),t(rs),t(Na),t(Ya),t(os),t(Da),t(hs),t(Oa),t(gs),t(Ka),t(ds),t(at),t(us),t(tt),t(et),t(vs),t(nt),t(lt),t(Fs),t(mt),t(it),t(Ms),t(rt),t(ct),t($s),t(ot),t(ht),t(xs),t(dt),t(js),t(ut),t(yt),t(vt),t(ft),t(Cs),t(bt),t(Mt),t(wt),t($t),t(Tt),t(xt),t(Gs),t(jt),t(Jt),t(Ps),t(_t),t(zt),t(Zs),t(Ct),t(Vs),t(Ut),t(Ls),t(kt),t(Xt),t(Ws),t(Bt),t(It),t(Ss)),t(y),d(f,s),d(b,s),d(J,s),d(z,s),d(k,s),d(E,s),d(V,s),d(W,s),d(H,s),d(S,s),d(Y,s),d(K,s),d(ss,s),d(ns,s),d(is,s),d(cs,s),d(ys,s),d(fs,s),d(bs,s),d(ws,s),d(Ts,s),d(Js,s),d(_s,s),d(zs,s),d(Us,s),d(ks,s),d(Xs,s),d(Bs,s),d(Is,s),d(Rs,s),d(Es,s),d(As,s),d(Qs,s)}}}const sn='{"title":"Comprensión avanzada de Group Relative Policy Optimization (GRPO) en DeepSeekMath","local":"comprensión-avanzada-de-group-relative-policy-optimization-grpo-en-deepseekmath","sections":[{"title":"El algoritmo GRPO","local":"el-algoritmo-grpo","sections":[{"title":"Paso 1: muestreo de grupos","local":"paso-1-muestreo-de-grupos","sections":[{"title":"Ejemplo","local":"ejemplo","sections":[],"depth":4}],"depth":3},{"title":"Paso 2: cálculo de la ventaja","local":"paso-2-cálculo-de-la-ventaja","sections":[{"title":"Distribución de recompensas","local":"distribución-de-recompensas","sections":[],"depth":4},{"title":"Fórmula de la ventaja","local":"fórmula-de-la-ventaja","sections":[],"depth":4},{"title":"Ejemplo","local":"ejemplo","sections":[],"depth":4},{"title":"Interpretación","local":"interpretación","sections":[],"depth":4}],"depth":3},{"title":"Paso 3: actualización de la política","local":"paso-3-actualización-de-la-política","sections":[],"depth":3}],"depth":2},{"title":"Componentes clave de la función objetivo","local":"componentes-clave-de-la-función-objetivo","sections":[{"title":"1. Cociente de probabilidad","local":"1-cociente-de-probabilidad","sections":[],"depth":3},{"title":"2. Función de recorte","local":"2-función-de-recorte","sections":[{"title":"Ejemplo con ( \\epsilon = 0.2 )","local":"ejemplo-con--epsilon--02-","sections":[],"depth":4}],"depth":3},{"title":"3. Divergencia KL","local":"3-divergencia-kl","sections":[],"depth":3}],"depth":2},{"title":"Ejemplo resuelto con GRPO","local":"ejemplo-resuelto-con-grpo","sections":[{"title":"Problema de ejemplo","local":"problema-de-ejemplo","sections":[],"depth":3},{"title":"Paso 1: muestreo de grupos","local":"paso-1-muestreo-de-grupos","sections":[],"depth":3},{"title":"Paso 2: cálculo de la ventaja","local":"paso-2-cálculo-de-la-ventaja","sections":[],"depth":3},{"title":"Paso 3: actualización de la política","local":"paso-3-actualización-de-la-política","sections":[],"depth":3}],"depth":2},{"title":"Ejemplo de implementación","local":"ejemplo-de-implementación","sections":[{"title":"1. Cargar el modelo y generar respuestas","local":"1-cargar-el-modelo-y-generar-respuestas","sections":[],"depth":3},{"title":"2. Calcular recompensas","local":"2-calcular-recompensas","sections":[],"depth":3},{"title":"3. Actualizar la política","local":"3-actualizar-la-política","sections":[],"depth":3}],"depth":2},{"title":"Resumen y próximos pasos","local":"resumen-y-próximos-pasos","sections":[],"depth":2},{"title":"Referencias","local":"referencias","sections":[],"depth":2}],"depth":1}';function an(qt){return qe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pn extends Se{constructor(y){super(),Ne(this,y,an,Ke,He,{})}}export{pn as component};