Buckets:
| import{s as jl,a as De,n as fl,o as bl}from"../chunks/scheduler.505acc25.js";import{S as hl,i as Cl,e as p,s as a,c as o,h as Il,a as r,d as s,b as n,f as re,g as m,j as w,k as i,l as gl,m as t,n as M,t as c,o as y,p as d}from"../chunks/index.e22abd30.js";import{C as $l,H as U,E as Gl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.a144e953.js";import{C as ae}from"../chunks/CodeBlock.f6688f67.js";function Bl(Ke){let j,ie,ne,oe,b,me,h,Me,C,Oe="En esta página aprenderemos a implementar Group Relative Policy Optimization (GRPO) usando la librería Transformer Reinforcement Learning (TRL). Nos centraremos en una implementación práctica con el mínimo código.",ce,I,el="Exploraremos los conceptos centrales de GRPO tal como aparecen en <code>GRPOTrainer</code> de TRL, usando fragmentos de la documentación oficial de TRL como guía.",ye,f,ll='<p>Este capítulo está orientado a quienes empiezan con TRL. Si ya tienes experiencia, quizá también te interese revisar la <a href="https://github.com/huggingface/open-r1/blob/main/src/open_r1/grpo.py" rel="nofollow">implementación de Open R1</a> de GRPO.</p>',de,g,sl="Primero, recordemos algunos conceptos importantes del algoritmo GRPO:",we,$,tl="<li>Formación de grupos: el modelo genera múltiples completions para cada prompt.</li> <li>Aprendizaje por preferencias: el modelo aprende a partir de una función de recompensa que compara grupos de completions.</li> <li>Configuración de entrenamiento: el modelo usa una configuración para controlar el proceso de entrenamiento.</li>",ue,G,al="¿Qué necesitamos para implementarlo?",Je,B,nl="<li>Definir un dataset de prompts.</li> <li>Definir una función de recompensa que reciba una lista de completions y devuelva una lista de recompensas.</li> <li>Configurar el entrenamiento con <code>GRPOConfig</code>.</li> <li>Entrenar el modelo con <code>GRPOTrainer</code>.</li>",Te,R,pl="Aquí tienes un ejemplo mínimo:",Ue,Z,je,k,fe,_,be,v,rl="Tu dataset debe contener prompts a los que el modelo responderá. <code>GRPOTrainer</code> generará múltiples completions para cada prompt y usará la función de recompensa para compararlas.",he,X,Ce,E,il="La función de recompensa es crucial: determina cómo aprende el modelo. Aquí tienes dos ejemplos prácticos:",Ie,x,ge,A,$e,F,ol="Parámetros clave a considerar en <code>GRPOConfig</code>:",Ge,Y,Be,z,ml="El parámetro <code>num_generation</code> es especialmente importante porque define el tamaño del grupo.",Re,Q,Ml="<li>Demasiado pequeño, por ejemplo 2 o 3, puede no aportar suficiente diversidad.</li> <li>Recomendado, entre 4 y 16, suele dar un buen equilibrio entre diversidad y coste.</li> <li>Valores más altos pueden mejorar el aprendizaje, pero incrementan bastante el coste computacional.</li>",Ze,V,ke,N,cl="<li><strong>Gestión de memoria</strong>: ajusta <code>per_device_train_batch_size</code> y <code>gradient_accumulation_steps</code> según la memoria de tu GPU.</li> <li><strong>Velocidad</strong>: activa <code>use_vllm=True</code> para acelerar la generación si tu modelo es compatible.</li> <li><strong>Monitorización</strong>: observa métricas como <code>reward</code>, <code>reward_std</code> y <code>kl</code> durante el entrenamiento.</li>",_e,W,ve,S,yl="El paper de DeepSeek R1 muestra varios enfoques eficaces que puedes adaptar.",Xe,q,Ee,H,xe,L,dl="Esta función penaliza completions demasiado cortas o demasiado largas y empuja al modelo hacia una longitud objetivo.",Ae,u,wl,Fe,P,Ye,D,ze,J,ul,Qe,K,Ve,O,Ne,T,Jl,We,ee,Tl="Estos ejemplos muestran cómo puedes implementar funciones de recompensa inspiradas en el proceso de entrenamiento de DeepSeek R1.",Se,le,qe,se,Ul="En la siguiente sección seguirás un ejercicio para implementar GRPO en TRL.",He,te,Le,pe,Pe;return b=new $l({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),h=new U({props:{title:"Implementar GRPO en TRL",local:"implementar-grpo-en-trl",headingTag:"h1"}}),Z=new ae({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTJDJTIwR1JQT0NvbmZpZyUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQSUwQSUyMyUyMDEuJTIwTG9hZCUyMHlvdXIlMjBkYXRhc2V0JTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ5b3VyX2RhdGFzZXQlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQSUwQSUyMyUyMDIuJTIwRGVmaW5lJTIwYSUyMHNpbXBsZSUyMHJld2FyZCUyMGZ1bmN0aW9uJTBBZGVmJTIwcmV3YXJkX2Z1bmMoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjJFeGFtcGxlJTNBJTIwUmV3YXJkJTIwbG9uZ2VyJTIwY29tcGxldGlvbnMlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmbG9hdChsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMEElMEElMjMlMjAzLiUyMENvbmZpZ3VyZSUyMHRyYWluaW5nJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm91dHB1dCUyMiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QzJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDIlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEMTAlMkMlMEEpJTBBJTBBJTIzJTIwNC4lMjBJbml0aWFsaXplJTIwYW5kJTIwdHJhaW4lMEF0cmFpbmVyJTIwJTNEJTIwR1JQT1RyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJ5b3VyX21vZGVsJTIyJTJDJTIwJTIwJTIzJTIwZS5nLiUyMCUyMlF3ZW4lMkZRd2VuMi0wLjVCLUluc3RydWN0JTIyJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRHJld2FyZF9mdW5jJTJDJTBBKSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer, GRPOConfig | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-comment"># 1. Load your dataset</span> | |
| dataset = load_dataset(<span class="hljs-string">"your_dataset"</span>, split=<span class="hljs-string">"train"</span>) | |
| <span class="hljs-comment"># 2. Define a simple reward function</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">completions, **kwargs</span>): | |
| <span class="hljs-string">"""Example: Reward longer completions"""</span> | |
| <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions] | |
| <span class="hljs-comment"># 3. Configure training</span> | |
| training_args = GRPOConfig( | |
| output_dir=<span class="hljs-string">"output"</span>, | |
| num_train_epochs=<span class="hljs-number">3</span>, | |
| per_device_train_batch_size=<span class="hljs-number">4</span>, | |
| gradient_accumulation_steps=<span class="hljs-number">2</span>, | |
| logging_steps=<span class="hljs-number">10</span>, | |
| ) | |
| <span class="hljs-comment"># 4. Initialize and train</span> | |
| trainer = GRPOTrainer( | |
| model=<span class="hljs-string">"your_model"</span>, <span class="hljs-comment"># e.g. "Qwen/Qwen2-0.5B-Instruct"</span> | |
| args=training_args, | |
| train_dataset=dataset, | |
| reward_funcs=reward_func, | |
| ) | |
| trainer.train()`,wrap:!1}}),k=new U({props:{title:"Componentes clave",local:"componentes-clave",headingTag:"h2"}}),_=new U({props:{title:"1. Formato del dataset",local:"1-formato-del-dataset",headingTag:"h3"}}),X=new U({props:{title:"2. Función de recompensa",local:"2-función-de-recompensa",headingTag:"h3"}}),x=new ae({props:{code:"JTIzJTIwRXhhbXBsZSUyMDElM0ElMjBSZXdhcmQlMjBiYXNlZCUyMG9uJTIwY29tcGxldGlvbiUyMGxlbmd0aCUwQWRlZiUyMHJld2FyZF9sZW5ndGgoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmbG9hdChsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMEElMEElMjMlMjBFeGFtcGxlJTIwMiUzQSUyMFJld2FyZCUyMGJhc2VkJTIwb24lMjBtYXRjaGluZyUyMGElMjBwYXR0ZXJuJTBBaW1wb3J0JTIwcmUlMEElMEElMEFkZWYlMjByZXdhcmRfZm9ybWF0KGNvbXBsZXRpb25zJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwcGF0dGVybiUyMCUzRCUyMHIlMjIlNUUlM0N0aGluayUzRS4qJTNGJTNDJTJGdGhpbmslM0UlM0NhbnN3ZXIlM0UuKiUzRiUzQyUyRmFuc3dlciUzRSUyNCUyMiUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QjEuMCUyMGlmJTIwcmUubWF0Y2gocGF0dGVybiUyQyUyMGMpJTIwZWxzZSUyMDAuMCUyMGZvciUyMGMlMjBpbiUyMGNvbXBsZXRpb25zJTVE",highlighted:`<span class="hljs-comment"># Example 1: Reward based on completion length</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_length</span>(<span class="hljs-params">completions, **kwargs</span>): | |
| <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions] | |
| <span class="hljs-comment"># Example 2: Reward based on matching a pattern</span> | |
| <span class="hljs-keyword">import</span> re | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_format</span>(<span class="hljs-params">completions, **kwargs</span>): | |
| pattern = <span class="hljs-string">r"^<think>.*?</think><answer>.*?</answer>$"</span> | |
| <span class="hljs-keyword">return</span> [<span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> re.<span class="hljs-keyword">match</span>(pattern, c) <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),A=new U({props:{title:"3. Configuración del entrenamiento",local:"3-configuración-del-entrenamiento",headingTag:"h3"}}),Y=new ae({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm91dHB1dCUyMiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QzJTJDJTBBJTIwJTIwJTIwJTIwbnVtX2dlbmVyYXRpb24lM0Q0JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMWUtNSUyQyUwQSUyMCUyMCUyMCUyMGxvZ2dpbmdfc3RlcHMlM0QxMCUyQyUwQSUyMCUyMCUyMCUyMHVzZV92bGxtJTNEVHJ1ZSUyQyUwQSk=",highlighted:`training_args = GRPOConfig( | |
| output_dir=<span class="hljs-string">"output"</span>, | |
| num_train_epochs=<span class="hljs-number">3</span>, | |
| num_generation=<span class="hljs-number">4</span>, | |
| per_device_train_batch_size=<span class="hljs-number">4</span>, | |
| gradient_accumulation_steps=<span class="hljs-number">2</span>, | |
| learning_rate=<span class="hljs-number">1e-5</span>, | |
| logging_steps=<span class="hljs-number">10</span>, | |
| use_vllm=<span class="hljs-literal">True</span>, | |
| )`,wrap:!1}}),V=new U({props:{title:"Consejos para tener éxito",local:"consejos-para-tener-éxito",headingTag:"h2"}}),W=new U({props:{title:"Diseño de funciones de recompensa",local:"diseño-de-funciones-de-recompensa",headingTag:"h2"}}),q=new U({props:{title:"1. Recompensas basadas en longitud",local:"1-recompensas-basadas-en-longitud",headingTag:"h3"}}),H=new ae({props:{code:"ZGVmJTIwcmV3YXJkX2xlbihjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMGlkZWFsX2xlbmd0aCUyMCUzRCUyMDIwJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTVCLWFicyhpZGVhbF9sZW5ndGglMjAtJTIwbGVuKGNvbXBsZXRpb24pKSUyMGZvciUyMGNvbXBsZXRpb24lMjBpbiUyMGNvbXBsZXRpb25zJTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_len</span>(<span class="hljs-params">completions, **kwargs</span>): | |
| ideal_length = <span class="hljs-number">20</span> | |
| <span class="hljs-keyword">return</span> [-<span class="hljs-built_in">abs</span>(ideal_length - <span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),P=new U({props:{title:"2. Recompensas basadas en reglas para tareas verificables",local:"2-recompensas-basadas-en-reglas-para-tareas-verificables",headingTag:"h2"}}),D=new ae({props:{code:"ZGVmJTIwcHJvYmxlbV9yZXdhcmQoY29tcGxldGlvbnMlMkMlMjBhbnN3ZXJzJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyUmV3YXJkJTIwZnVuY3Rpb24lMjBmb3IlMjBtYXRoJTIwcHJvYmxlbXMlMjB3aXRoJTIwdmVyaWZpYWJsZSUyMGFuc3dlcnMlMEElMjAlMjAlMjAlMjBjb21wbGV0aW9ucyUzQSUyMGxpc3QlMjBvZiUyMGNvbXBsZXRpb25zJTIwdG8lMjBldmFsdWF0ZSUwQSUyMCUyMCUyMCUyMGFuc3dlcnMlM0ElMjBsaXN0JTIwb2YlMjBhbnN3ZXJzJTIwdG8lMjB0aGUlMjBwcm9ibGVtcyUyMGZyb20lMjB0aGUlMjBkYXRhc2V0JTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTBBJTIwJTIwJTIwJTIwcmV3YXJkcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGNvbXBsZXRpb24lMkMlMjBjb3JyZWN0X2Fuc3dlciUyMGluJTIwemlwKGNvbXBsZXRpb25zJTJDJTIwYW5zd2VycyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXIlMjAlM0QlMjBleHRyYWN0X2ZpbmFsX2Fuc3dlcihjb21wbGV0aW9uKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZCUyMCUzRCUyMDEuMCUyMGlmJTIwYW5zd2VyJTIwJTNEJTNEJTIwY29ycmVjdF9hbnN3ZXIlMjBlbHNlJTIwMC4wJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV3YXJkcy5hcHBlbmQocmV3YXJkKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV4Y2VwdCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZHMuYXBwZW5kKDAuMCklMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjByZXdhcmRz",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">problem_reward</span>(<span class="hljs-params">completions, answers, **kwargs</span>): | |
| <span class="hljs-string">"""Reward function for math problems with verifiable answers | |
| completions: list of completions to evaluate | |
| answers: list of answers to the problems from the dataset | |
| """</span> | |
| rewards = [] | |
| <span class="hljs-keyword">for</span> completion, correct_answer <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(completions, answers): | |
| <span class="hljs-keyword">try</span>: | |
| answer = extract_final_answer(completion) | |
| reward = <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> answer == correct_answer <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> | |
| rewards.append(reward) | |
| <span class="hljs-keyword">except</span>: | |
| rewards.append(<span class="hljs-number">0.0</span>) | |
| <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),K=new U({props:{title:"3. Recompensas basadas en formato",local:"3-recompensas-basadas-en-formato",headingTag:"h2"}}),O=new ae({props:{code:"ZGVmJTIwZm9ybWF0X3Jld2FyZChjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMlJld2FyZCUyMGNvbXBsZXRpb25zJTIwdGhhdCUyMGZvbGxvdyUyMHRoZSUyMGRlc2lyZWQlMjBmb3JtYXQlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjBwYXR0ZXJuJTIwJTNEJTIwciUyMiUzQ3RoaW5rJTNFKC4qJTNGKSUzQyUyRnRoaW5rJTNFJTVDcyolM0NhbnN3ZXIlM0UoLiolM0YpJTNDJTJGYW5zd2VyJTNFJTIyJTBBJTBBJTIwJTIwJTIwJTIwcmV3YXJkcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGNvbXBsZXRpb24lMjBpbiUyMGNvbXBsZXRpb25zJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWF0Y2glMjAlM0QlMjByZS5zZWFyY2gocGF0dGVybiUyQyUyMGNvbXBsZXRpb24lMkMlMjByZS5ET1RBTEwpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBtYXRjaCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRoaW5rX2NvbnRlbnQlMjAlM0QlMjBtYXRjaC5ncm91cCgxKS5zdHJpcCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYW5zd2VyX2NvbnRlbnQlMjAlM0QlMjBtYXRjaC5ncm91cCgyKS5zdHJpcCgpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBsZW4odGhpbmtfY29udGVudCklMjAlM0UlMjAyMCUyMGFuZCUyMGxlbihhbnN3ZXJfY29udGVudCklMjAlM0UlMjAwJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV3YXJkcy5hcHBlbmQoMS4wKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZCgwLjUpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZHMuYXBwZW5kKDAuMCklMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjByZXdhcmRz",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">format_reward</span>(<span class="hljs-params">completions, **kwargs</span>): | |
| <span class="hljs-string">"""Reward completions that follow the desired format"""</span> | |
| pattern = <span class="hljs-string">r"<think>(.*?)</think>\\s*<answer>(.*?)</answer>"</span> | |
| rewards = [] | |
| <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions: | |
| <span class="hljs-keyword">match</span> = re.search(pattern, completion, re.DOTALL) | |
| <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span>: | |
| think_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">1</span>).strip() | |
| answer_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">2</span>).strip() | |
| <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(think_content) > <span class="hljs-number">20</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(answer_content) > <span class="hljs-number">0</span>: | |
| rewards.append(<span class="hljs-number">1.0</span>) | |
| <span class="hljs-keyword">else</span>: | |
| rewards.append(<span class="hljs-number">0.5</span>) | |
| <span class="hljs-keyword">else</span>: | |
| rewards.append(<span class="hljs-number">0.0</span>) | |
| <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),le=new U({props:{title:"¡Eso es todo!",local:"eso-es-todo",headingTag:"h2"}}),te=new Gl({props:{source:"https://github.com/huggingface/course/blob/main/chapters/es/chapter12/4.mdx"}}),{c(){j=p("meta"),ie=a(),ne=p("p"),oe=a(),o(b.$$.fragment),me=a(),o(h.$$.fragment),Me=a(),C=p("p"),C.textContent=Oe,ce=a(),I=p("p"),I.innerHTML=el,ye=a(),f=p("blockquote"),f.innerHTML=ll,de=a(),g=p("p"),g.textContent=sl,we=a(),$=p("ul"),$.innerHTML=tl,ue=a(),G=p("p"),G.textContent=al,Je=a(),B=p("ul"),B.innerHTML=nl,Te=a(),R=p("p"),R.textContent=pl,Ue=a(),o(Z.$$.fragment),je=a(),o(k.$$.fragment),fe=a(),o(_.$$.fragment),be=a(),v=p("p"),v.innerHTML=rl,he=a(),o(X.$$.fragment),Ce=a(),E=p("p"),E.textContent=il,Ie=a(),o(x.$$.fragment),ge=a(),o(A.$$.fragment),$e=a(),F=p("p"),F.innerHTML=ol,Ge=a(),o(Y.$$.fragment),Be=a(),z=p("p"),z.innerHTML=ml,Re=a(),Q=p("ul"),Q.innerHTML=Ml,Ze=a(),o(V.$$.fragment),ke=a(),N=p("ol"),N.innerHTML=cl,_e=a(),o(W.$$.fragment),ve=a(),S=p("p"),S.textContent=yl,Xe=a(),o(q.$$.fragment),Ee=a(),o(H.$$.fragment),xe=a(),L=p("p"),L.textContent=dl,Ae=a(),u=p("iframe"),Fe=a(),o(P.$$.fragment),Ye=a(),o(D.$$.fragment),ze=a(),J=p("iframe"),Qe=a(),o(K.$$.fragment),Ve=a(),o(O.$$.fragment),Ne=a(),T=p("iframe"),We=a(),ee=p("p"),ee.textContent=Tl,Se=a(),o(le.$$.fragment),qe=a(),se=p("p"),se.textContent=Ul,He=a(),o(te.$$.fragment),Le=a(),pe=p("p"),this.h()},l(e){const l=Il("svelte-u9bgzb",document.head);j=r(l,"META",{name:!0,content:!0}),l.forEach(s),ie=n(e),ne=r(e,"P",{}),re(ne).forEach(s),oe=n(e),m(b.$$.fragment,e),me=n(e),m(h.$$.fragment,e),Me=n(e),C=r(e,"P",{"data-svelte-h":!0}),w(C)!=="svelte-927fg4"&&(C.textContent=Oe),ce=n(e),I=r(e,"P",{"data-svelte-h":!0}),w(I)!=="svelte-1frme0"&&(I.innerHTML=el),ye=n(e),f=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),w(f)!=="svelte-11norxp"&&(f.innerHTML=ll),de=n(e),g=r(e,"P",{"data-svelte-h":!0}),w(g)!=="svelte-1ajhojj"&&(g.textContent=sl),we=n(e),$=r(e,"UL",{"data-svelte-h":!0}),w($)!=="svelte-12kd6qs"&&($.innerHTML=tl),ue=n(e),G=r(e,"P",{"data-svelte-h":!0}),w(G)!=="svelte-1dqejmx"&&(G.textContent=al),Je=n(e),B=r(e,"UL",{"data-svelte-h":!0}),w(B)!=="svelte-9mzu1v"&&(B.innerHTML=nl),Te=n(e),R=r(e,"P",{"data-svelte-h":!0}),w(R)!=="svelte-uessia"&&(R.textContent=pl),Ue=n(e),m(Z.$$.fragment,e),je=n(e),m(k.$$.fragment,e),fe=n(e),m(_.$$.fragment,e),be=n(e),v=r(e,"P",{"data-svelte-h":!0}),w(v)!=="svelte-149cq7e"&&(v.innerHTML=rl),he=n(e),m(X.$$.fragment,e),Ce=n(e),E=r(e,"P",{"data-svelte-h":!0}),w(E)!=="svelte-1mr8kty"&&(E.textContent=il),Ie=n(e),m(x.$$.fragment,e),ge=n(e),m(A.$$.fragment,e),$e=n(e),F=r(e,"P",{"data-svelte-h":!0}),w(F)!=="svelte-8gixmy"&&(F.innerHTML=ol),Ge=n(e),m(Y.$$.fragment,e),Be=n(e),z=r(e,"P",{"data-svelte-h":!0}),w(z)!=="svelte-1y5kihb"&&(z.innerHTML=ml),Re=n(e),Q=r(e,"UL",{"data-svelte-h":!0}),w(Q)!=="svelte-4btybj"&&(Q.innerHTML=Ml),Ze=n(e),m(V.$$.fragment,e),ke=n(e),N=r(e,"OL",{"data-svelte-h":!0}),w(N)!=="svelte-17xjn5k"&&(N.innerHTML=cl),_e=n(e),m(W.$$.fragment,e),ve=n(e),S=r(e,"P",{"data-svelte-h":!0}),w(S)!=="svelte-vmayqv"&&(S.textContent=yl),Xe=n(e),m(q.$$.fragment,e),Ee=n(e),m(H.$$.fragment,e),xe=n(e),L=r(e,"P",{"data-svelte-h":!0}),w(L)!=="svelte-6dzchn"&&(L.textContent=dl),Ae=n(e),u=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),re(u).forEach(s),Fe=n(e),m(P.$$.fragment,e),Ye=n(e),m(D.$$.fragment,e),ze=n(e),J=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),re(J).forEach(s),Qe=n(e),m(K.$$.fragment,e),Ve=n(e),m(O.$$.fragment,e),Ne=n(e),T=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),re(T).forEach(s),We=n(e),ee=r(e,"P",{"data-svelte-h":!0}),w(ee)!=="svelte-12j9xcv"&&(ee.textContent=Tl),Se=n(e),m(le.$$.fragment,e),qe=n(e),se=r(e,"P",{"data-svelte-h":!0}),w(se)!=="svelte-249vsf"&&(se.textContent=Ul),He=n(e),m(te.$$.fragment,e),Le=n(e),pe=r(e,"P",{}),re(pe).forEach(s),this.h()},h(){i(j,"name","hf:doc:metadata"),i(j,"content",Rl),i(f,"class","tip"),De(u.src,wl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_length.py&embed=true&show-chrome=false")||i(u,"src",wl),i(u,"title","Marimo Notebook"),i(u,"width","100%"),i(u,"height","800px"),i(u,"frameborder","0"),i(u,"allow","clipboard-write"),De(J.src,ul="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_math.py&embed=true&show-chrome=false")||i(J,"src",ul),i(J,"title","Marimo Notebook"),i(J,"width","100%"),i(J,"height","800px"),i(J,"frameborder","0"),i(J,"allow","clipboard-write"),De(T.src,Jl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_format.py&embed=true&show-chrome=false")||i(T,"src",Jl),i(T,"title","Marimo Notebook"),i(T,"width","100%"),i(T,"height","800px"),i(T,"frameborder","0"),i(T,"allow","clipboard-write")},m(e,l){gl(document.head,j),t(e,ie,l),t(e,ne,l),t(e,oe,l),M(b,e,l),t(e,me,l),M(h,e,l),t(e,Me,l),t(e,C,l),t(e,ce,l),t(e,I,l),t(e,ye,l),t(e,f,l),t(e,de,l),t(e,g,l),t(e,we,l),t(e,$,l),t(e,ue,l),t(e,G,l),t(e,Je,l),t(e,B,l),t(e,Te,l),t(e,R,l),t(e,Ue,l),M(Z,e,l),t(e,je,l),M(k,e,l),t(e,fe,l),M(_,e,l),t(e,be,l),t(e,v,l),t(e,he,l),M(X,e,l),t(e,Ce,l),t(e,E,l),t(e,Ie,l),M(x,e,l),t(e,ge,l),M(A,e,l),t(e,$e,l),t(e,F,l),t(e,Ge,l),M(Y,e,l),t(e,Be,l),t(e,z,l),t(e,Re,l),t(e,Q,l),t(e,Ze,l),M(V,e,l),t(e,ke,l),t(e,N,l),t(e,_e,l),M(W,e,l),t(e,ve,l),t(e,S,l),t(e,Xe,l),M(q,e,l),t(e,Ee,l),M(H,e,l),t(e,xe,l),t(e,L,l),t(e,Ae,l),t(e,u,l),t(e,Fe,l),M(P,e,l),t(e,Ye,l),M(D,e,l),t(e,ze,l),t(e,J,l),t(e,Qe,l),M(K,e,l),t(e,Ve,l),M(O,e,l),t(e,Ne,l),t(e,T,l),t(e,We,l),t(e,ee,l),t(e,Se,l),M(le,e,l),t(e,qe,l),t(e,se,l),t(e,He,l),M(te,e,l),t(e,Le,l),t(e,pe,l),Pe=!0},p:fl,i(e){Pe||(c(b.$$.fragment,e),c(h.$$.fragment,e),c(Z.$$.fragment,e),c(k.$$.fragment,e),c(_.$$.fragment,e),c(X.$$.fragment,e),c(x.$$.fragment,e),c(A.$$.fragment,e),c(Y.$$.fragment,e),c(V.$$.fragment,e),c(W.$$.fragment,e),c(q.$$.fragment,e),c(H.$$.fragment,e),c(P.$$.fragment,e),c(D.$$.fragment,e),c(K.$$.fragment,e),c(O.$$.fragment,e),c(le.$$.fragment,e),c(te.$$.fragment,e),Pe=!0)},o(e){y(b.$$.fragment,e),y(h.$$.fragment,e),y(Z.$$.fragment,e),y(k.$$.fragment,e),y(_.$$.fragment,e),y(X.$$.fragment,e),y(x.$$.fragment,e),y(A.$$.fragment,e),y(Y.$$.fragment,e),y(V.$$.fragment,e),y(W.$$.fragment,e),y(q.$$.fragment,e),y(H.$$.fragment,e),y(P.$$.fragment,e),y(D.$$.fragment,e),y(K.$$.fragment,e),y(O.$$.fragment,e),y(le.$$.fragment,e),y(te.$$.fragment,e),Pe=!1},d(e){e&&(s(ie),s(ne),s(oe),s(me),s(Me),s(C),s(ce),s(I),s(ye),s(f),s(de),s(g),s(we),s($),s(ue),s(G),s(Je),s(B),s(Te),s(R),s(Ue),s(je),s(fe),s(be),s(v),s(he),s(Ce),s(E),s(Ie),s(ge),s($e),s(F),s(Ge),s(Be),s(z),s(Re),s(Q),s(Ze),s(ke),s(N),s(_e),s(ve),s(S),s(Xe),s(Ee),s(xe),s(L),s(Ae),s(u),s(Fe),s(Ye),s(ze),s(J),s(Qe),s(Ve),s(Ne),s(T),s(We),s(ee),s(Se),s(qe),s(se),s(He),s(Le),s(pe)),s(j),d(b,e),d(h,e),d(Z,e),d(k,e),d(_,e),d(X,e),d(x,e),d(A,e),d(Y,e),d(V,e),d(W,e),d(q,e),d(H,e),d(P,e),d(D,e),d(K,e),d(O,e),d(le,e),d(te,e)}}}const Rl='{"title":"Implementar GRPO en TRL","local":"implementar-grpo-en-trl","sections":[{"title":"Componentes clave","local":"componentes-clave","sections":[{"title":"1. Formato del dataset","local":"1-formato-del-dataset","sections":[],"depth":3},{"title":"2. Función de recompensa","local":"2-función-de-recompensa","sections":[],"depth":3},{"title":"3. Configuración del entrenamiento","local":"3-configuración-del-entrenamiento","sections":[],"depth":3}],"depth":2},{"title":"Consejos para tener éxito","local":"consejos-para-tener-éxito","sections":[],"depth":2},{"title":"Diseño de funciones de recompensa","local":"diseño-de-funciones-de-recompensa","sections":[{"title":"1. Recompensas basadas en longitud","local":"1-recompensas-basadas-en-longitud","sections":[],"depth":3}],"depth":2},{"title":"2. Recompensas basadas en reglas para tareas verificables","local":"2-recompensas-basadas-en-reglas-para-tareas-verificables","sections":[],"depth":2},{"title":"3. Recompensas basadas en formato","local":"3-recompensas-basadas-en-formato","sections":[],"depth":2},{"title":"¡Eso es todo!","local":"eso-es-todo","sections":[],"depth":2}],"depth":1}';function Zl(Ke){return bl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class El extends hl{constructor(j){super(),Cl(this,j,Zl,Bl,jl,{})}}export{El as component}; | |
Xet Storage Details
- Size:
- 25.8 kB
- Xet hash:
- c34fb7c4576181cd5eda99c05f72ac815813576ab2d07929483e2e969ffccbb6
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.