course/pr_1194/en/_app/immutable/nodes/32.c8a3bb79.js

import{s as kl,a as nl,n as Al,o as vl}from"../chunks/scheduler.893fe8c9.js";import{S as xl,i as _l,e as i,s as a,c as o,h as Xl,a as r,d as t,b as n,f as me,g as m,j as p,k as M,l as El,m as s,n as y,t as w,o as c,p as d}from"../chunks/index.b1df2166.js";import{C as Wl,H as j,E as Fl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.df2177af.js";import{C as pe}from"../chunks/CodeBlock.cc3a1294.js";function Yl(il){let u,ye,Me,we,f,ce,C,de,I,rl="In this page, we’ll learn how to implement Group Relative Policy Optimization (GRPO) using the Transformer Reinforcement Learning (TRL) library. We’ll focus on practical implementation with minimal code.",Ue,b,pl="We’ll explore the core concepts of GRPO as they are embodied in TRL’s GRPOTrainer, using snippets from the official TRL documentation to guide us.",Je,h,Ml='<p>This chapter is aimed at TRL beginners. If you are already familiar with TRL, you might want to also check out the <a href="https://github.com/huggingface/open-r1/blob/main/src/open_r1/grpo.py" rel="nofollow">Open R1 implementation</a> of GRPO.</p>',Te,g,ol="First, let’s remind ourselves of some of the important concepts of GRPO algorithm:",je,G,ml="<li>Group Formation: The model generates multiple completions for each prompt.</li> <li>Preference Learning: The model learns from a reward function that compares groups of completions.</li> <li>Training Configuration: The model uses a configuration to control the training process.</li>",ue,B,yl="What do we need to do to implement GRPO?",he,Z,wl="<li>Define a dataset of prompts.</li> <li>Define a reward function that takes a list of completions and returns a list of rewards.</li> <li>Configure the training process with a GRPOConfig.</li> <li>Train the model using the GRPOTrainer.</li>",fe,$,cl="Here’s a minimal example to get started with GRPO training:",Ce,R,Ie,k,be,A,ge,v,dl="Your dataset should contain prompts that the model will respond to. The GRPO trainer will generate multiple completions for each prompt and use the reward function to compare them.",Ge,x,Be,_,Ul="The reward function is crucial - it determines how the model learns. Here are two practical examples:",Ze,X,$e,E,Re,W,Jl="Key parameters to consider in <code>GRPOConfig</code>:",ke,F,Ae,Y,Tl="The <code>num_generation</code> parameter is particularly important for GRPO as it defines the group size - how many different completions the model will generate for each prompt. This is a key differentiator from other RL methods:",ve,S,jl="<li>Too small (e.g., 2-3): May not provide enough diversity for meaningful comparisons</li> <li>Recommended (4-16): Provides good balance between diversity and computational efficiency</li> <li>Larger values: May improve learning but significantly increases computational cost</li>",xe,z,ul="The group size should be chosen based on your computational resources and the complexity of your task. For simple tasks, smaller groups (4-8) may be sufficient, while more complex reasoning tasks might benefit from larger groups (8-16).",_e,Q,Xe,V,hl=`<li><strong>Memory Management</strong>: Adjust <code>per_device_train_batch_size</code> and <code>gradient_accumulation_steps</code> based on your GPU memory.</li> <li><strong>Speed</strong>: Enable <code>use_vllm=True</code> for faster generation if your model is supported.</li> <li><strong>Monitoring</strong>: Watch the logged metrics during training:
<ul><li><code>reward</code>: Average reward across completions</li> <li><code>reward_std</code>: Standard deviation within reward groups</li> <li><code>kl</code>: KL divergence from reference model</li></ul></li>`,Ee,N,We,H,fl="The DeepSeek R1 paper demonstrates several effective approaches to reward function design that you can adapt for your own GRPO implementation:",Fe,L,Ye,q,Cl="One of the easiest rewards to implement is a length-based reward. You can reward longer completions:",Se,P,ze,D,Il="This reward function penalizes completions that are too short or too long, encouraging the model to generate completions that are close to the ideal length of 20 tokens.",Qe,U,bl,Ve,K,Ne,O,gl="For tasks with objectively correct answers (like mathematics or coding), you can implement rule-based reward functions:",He,ee,Le,J,Gl,qe,le,Pe,te,Bl="You can also reward proper formatting, which was important in the DeepSeek R1 training:",De,se,Ke,T,Zl,Oe,ae,$l="These examples demonstrate how you can implement reward functions inspired by the DeepSeek R1 training process, focusing on correctness, formatting, and combined signals.",el,ne,ll,ie,Rl="In the next section, you will follow an exercise to implement GRPO in TRL.",tl,re,sl,oe,al;return f=new Wl({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),C=new j({props:{title:"Implementing GRPO in TRL",local:"implementing-grpo-in-trl",headingTag:"h1"}}),R=new pe({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTJDJTIwR1JQT0NvbmZpZyUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQSUwQSUyMyUyMDEuJTIwTG9hZCUyMHlvdXIlMjBkYXRhc2V0JTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ5b3VyX2RhdGFzZXQlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQSUwQSUyMyUyMDIuJTIwRGVmaW5lJTIwYSUyMHNpbXBsZSUyMHJld2FyZCUyMGZ1bmN0aW9uJTBBZGVmJTIwcmV3YXJkX2Z1bmMoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjJFeGFtcGxlJTNBJTIwUmV3YXJkJTIwbG9uZ2VyJTIwY29tcGxldGlvbnMlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmbG9hdChsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMEElMEElMjMlMjAzLiUyMENvbmZpZ3VyZSUyMHRyYWluaW5nJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm91dHB1dCUyMiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QzJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDIlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEMTAlMkMlMEEpJTBBJTBBJTIzJTIwNC4lMjBJbml0aWFsaXplJTIwYW5kJTIwdHJhaW4lMEF0cmFpbmVyJTIwJTNEJTIwR1JQT1RyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJ5b3VyX21vZGVsJTIyJTJDJTIwJTIwJTIzJTIwZS5nLiUyMCUyMlF3ZW4lMkZRd2VuMi0wLjVCLUluc3RydWN0JTIyJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRHJld2FyZF9mdW5jJTJDJTBBKSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer, GRPOConfig
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># 1. Load your dataset</span>
dataset = load_dataset(<span class="hljs-string">&quot;your_dataset&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)


<span class="hljs-comment"># 2. Define a simple reward function</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Example: Reward longer completions&quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]


<span class="hljs-comment"># 3. Configure training</span>
training_args = GRPOConfig(
    output_dir=<span class="hljs-string">&quot;output&quot;</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    per_device_train_batch_size=<span class="hljs-number">4</span>,
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    logging_steps=<span class="hljs-number">10</span>,
)

<span class="hljs-comment"># 4. Initialize and train</span>
trainer = GRPOTrainer(
    model=<span class="hljs-string">&quot;your_model&quot;</span>,  <span class="hljs-comment"># e.g. &quot;Qwen/Qwen2-0.5B-Instruct&quot;</span>
    args=training_args,
    train_dataset=dataset,
    reward_funcs=reward_func,
)
trainer.train()`,wrap:!1}}),k=new j({props:{title:"Key Components",local:"key-components",headingTag:"h2"}}),A=new j({props:{title:"1. Dataset Format",local:"1-dataset-format",headingTag:"h3"}}),x=new j({props:{title:"2. Reward Function",local:"2-reward-function",headingTag:"h3"}}),X=new pe({props:{code:"JTIzJTIwRXhhbXBsZSUyMDElM0ElMjBSZXdhcmQlMjBiYXNlZCUyMG9uJTIwY29tcGxldGlvbiUyMGxlbmd0aCUwQWRlZiUyMHJld2FyZF9sZW5ndGgoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmbG9hdChsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMEElMEElMjMlMjBFeGFtcGxlJTIwMiUzQSUyMFJld2FyZCUyMGJhc2VkJTIwb24lMjBtYXRjaGluZyUyMGElMjBwYXR0ZXJuJTBBaW1wb3J0JTIwcmUlMEElMEElMEFkZWYlMjByZXdhcmRfZm9ybWF0KGNvbXBsZXRpb25zJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwcGF0dGVybiUyMCUzRCUyMHIlMjIlNUUlM0N0aGluayUzRS4qJTNGJTNDJTJGdGhpbmslM0UlM0NhbnN3ZXIlM0UuKiUzRiUzQyUyRmFuc3dlciUzRSUyNCUyMiUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QjEuMCUyMGlmJTIwcmUubWF0Y2gocGF0dGVybiUyQyUyMGMpJTIwZWxzZSUyMDAuMCUyMGZvciUyMGMlMjBpbiUyMGNvbXBsZXRpb25zJTVE",highlighted:`<span class="hljs-comment"># Example 1: Reward based on completion length</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_length</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]


<span class="hljs-comment"># Example 2: Reward based on matching a pattern</span>
<span class="hljs-keyword">import</span> re


<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_format</span>(<span class="hljs-params">completions, **kwargs</span>):
    pattern = <span class="hljs-string">r&quot;^&lt;think&gt;.*?&lt;/think&gt;&lt;answer&gt;.*?&lt;/answer&gt;$&quot;</span>
    <span class="hljs-keyword">return</span> [<span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> re.<span class="hljs-keyword">match</span>(pattern, c) <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),E=new j({props:{title:"3. Training Configuration",local:"3-training-configuration",headingTag:"h3"}}),F=new pe({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwJTIzJTIwRXNzZW50aWFsJTIwcGFyYW1ldGVycyUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJvdXRwdXQlMjIlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMyUyQyUwQSUyMCUyMCUyMCUyMG51bV9nZW5lcmF0aW9uJTNENCUyQyUyMCUyMCUyMyUyME51bWJlciUyMG9mJTIwY29tcGxldGlvbnMlMjB0byUyMGdlbmVyYXRlJTIwZm9yJTIwZWFjaCUyMHByb21wdCUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDQlMkMlMjAlMjAlMjMlMjBXZSUyMHdhbnQlMjB0byUyMGdldCUyMGFsbCUyMGdlbmVyYXRpb25zJTIwaW4lMjBvbmUlMjBkZXZpY2UlMjBiYXRjaCUwQSUyMCUyMCUyMCUyMCUyMyUyME9wdGlvbmFsJTIwYnV0JTIwdXNlZnVsJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNEMiUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QxZS01JTJDJTBBJTIwJTIwJTIwJTIwbG9nZ2luZ19zdGVwcyUzRDEwJTJDJTBBJTIwJTIwJTIwJTIwJTIzJTIwR1JQTyUyMHNwZWNpZmljJTIwKG9wdGlvbmFsKSUwQSUyMCUyMCUyMCUyMHVzZV92bGxtJTNEVHJ1ZSUyQyUyMCUyMCUyMyUyMFNwZWVkJTIwdXAlMjBnZW5lcmF0aW9uJTBBKQ==",highlighted:`training_args = GRPOConfig(
    <span class="hljs-comment"># Essential parameters</span>
    output_dir=<span class="hljs-string">&quot;output&quot;</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    num_generation=<span class="hljs-number">4</span>,  <span class="hljs-comment"># Number of completions to generate for each prompt</span>
    per_device_train_batch_size=<span class="hljs-number">4</span>,  <span class="hljs-comment"># We want to get all generations in one device batch</span>
    <span class="hljs-comment"># Optional but useful</span>
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    learning_rate=<span class="hljs-number">1e-5</span>,
    logging_steps=<span class="hljs-number">10</span>,
    <span class="hljs-comment"># GRPO specific (optional)</span>
    use_vllm=<span class="hljs-literal">True</span>,  <span class="hljs-comment"># Speed up generation</span>
)`,wrap:!1}}),Q=new j({props:{title:"Tips for Success",local:"tips-for-success",headingTag:"h2"}}),N=new j({props:{title:"Reward Function Design",local:"reward-function-design",headingTag:"h2"}}),L=new j({props:{title:"1. Length-Based Rewards",local:"1-length-based-rewards",headingTag:"h3"}}),P=new pe({props:{code:"ZGVmJTIwcmV3YXJkX2xlbihjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMGlkZWFsX2xlbmd0aCUyMCUzRCUyMDIwJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTVCLWFicyhpZGVhbF9sZW5ndGglMjAtJTIwbGVuKGNvbXBsZXRpb24pKSUyMGZvciUyMGNvbXBsZXRpb24lMjBpbiUyMGNvbXBsZXRpb25zJTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_len</span>(<span class="hljs-params">completions, **kwargs</span>):
    ideal_length = <span class="hljs-number">20</span>
    <span class="hljs-keyword">return</span> [-<span class="hljs-built_in">abs</span>(ideal_length - <span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),K=new j({props:{title:"2. Rule-Based Rewards for Verifiable Tasks",local:"2-rule-based-rewards-for-verifiable-tasks",headingTag:"h2"}}),ee=new pe({props:{code:"ZGVmJTIwcHJvYmxlbV9yZXdhcmQoY29tcGxldGlvbnMlMkMlMjBhbnN3ZXJzJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyUmV3YXJkJTIwZnVuY3Rpb24lMjBmb3IlMjBtYXRoJTIwcHJvYmxlbXMlMjB3aXRoJTIwdmVyaWZpYWJsZSUyMGFuc3dlcnMlMEElMjAlMjAlMjAlMjBjb21wbGV0aW9ucyUzQSUyMGxpc3QlMjBvZiUyMGNvbXBsZXRpb25zJTIwdG8lMjBldmFsdWF0ZSUwQSUyMCUyMCUyMCUyMGFuc3dlcnMlM0ElMjBsaXN0JTIwb2YlMjBhbnN3ZXJzJTIwdG8lMjB0aGUlMjBwcm9ibGVtcyUyMGZyb20lMjB0aGUlMjBkYXRhc2V0JTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTBBJTIwJTIwJTIwJTIwcmV3YXJkcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGNvbXBsZXRpb24lMkMlMjBjb3JyZWN0X2Fuc3dlciUyMGluJTIwemlwKGNvbXBsZXRpb25zJTJDJTIwYW5zd2VycyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBFeHRyYWN0JTIwdGhlJTIwYW5zd2VyJTIwZnJvbSUyMHRoZSUyMGNvbXBsZXRpb24lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBUaGlzJTIwaXMlMjBhJTIwc2ltcGxpZmllZCUyMGV4YW1wbGUlMjAtJTIweW91J2QlMjBuZWVkJTIwcHJvcGVyJTIwcGFyc2luZyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFuc3dlciUyMCUzRCUyMGV4dHJhY3RfZmluYWxfYW5zd2VyKGNvbXBsZXRpb24pJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwQmluYXJ5JTIwcmV3YXJkJTNBJTIwMSUyMGZvciUyMGNvcnJlY3QlMkMlMjAwJTIwZm9yJTIwaW5jb3JyZWN0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV3YXJkJTIwJTNEJTIwMS4wJTIwaWYlMjBhbnN3ZXIlMjAlM0QlM0QlMjBjb3JyZWN0X2Fuc3dlciUyMGVsc2UlMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZChyZXdhcmQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXhjZXB0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwSWYlMjB3ZSUyMGNhbid0JTIwcGFyc2UlMjBhbiUyMGFuc3dlciUyQyUyMGdpdmUlMjBhJTIwbG93JTIwcmV3YXJkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV3YXJkcy5hcHBlbmQoMC4wKSUwQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJld2FyZHM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">problem_reward</span>(<span class="hljs-params">completions, answers, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Reward function for math problems with verifiable answers
    completions: list of completions to evaluate
    answers: list of answers to the problems from the dataset
    &quot;&quot;&quot;</span>

    rewards = []
    <span class="hljs-keyword">for</span> completion, correct_answer <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(completions, answers):
        <span class="hljs-comment"># Extract the answer from the completion</span>
        <span class="hljs-keyword">try</span>:
            <span class="hljs-comment"># This is a simplified example - you&#x27;d need proper parsing</span>
            answer = extract_final_answer(completion)
            <span class="hljs-comment"># Binary reward: 1 for correct, 0 for incorrect</span>
            reward = <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> answer == correct_answer <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span>
            rewards.append(reward)
        <span class="hljs-keyword">except</span>:
            <span class="hljs-comment"># If we can&#x27;t parse an answer, give a low reward</span>
            rewards.append(<span class="hljs-number">0.0</span>)

    <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),le=new j({props:{title:"3. Format-Based Rewards",local:"3-format-based-rewards",headingTag:"h2"}}),se=new pe({props:{code:"ZGVmJTIwZm9ybWF0X3Jld2FyZChjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMlJld2FyZCUyMGNvbXBsZXRpb25zJTIwdGhhdCUyMGZvbGxvdyUyMHRoZSUyMGRlc2lyZWQlMjBmb3JtYXQlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjAlMjMlMjBFeGFtcGxlJTNBJTIwQ2hlY2slMjBpZiUyMHRoZSUyMGNvbXBsZXRpb24lMjBmb2xsb3dzJTIwYSUyMHRoaW5rLXRoZW4tYW5zd2VyJTIwZm9ybWF0JTBBJTIwJTIwJTIwJTIwcGF0dGVybiUyMCUzRCUyMHIlMjIlM0N0aGluayUzRSguKiUzRiklM0MlMkZ0aGluayUzRSU1Q3MqJTNDYW5zd2VyJTNFKC4qJTNGKSUzQyUyRmFuc3dlciUzRSUyMiUwQSUwQSUyMCUyMCUyMCUyMHJld2FyZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBjb21wbGV0aW9uJTIwaW4lMjBjb21wbGV0aW9ucyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hdGNoJTIwJTNEJTIwcmUuc2VhcmNoKHBhdHRlcm4lMkMlMjBjb21wbGV0aW9uJTJDJTIwcmUuRE9UQUxMKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwbWF0Y2glM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBDaGVjayUyMGlmJTIwdGhlcmUncyUyMHN1YnN0YW50aWFsJTIwY29udGVudCUyMGluJTIwYm90aCUyMHNlY3Rpb25zJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdGhpbmtfY29udGVudCUyMCUzRCUyMG1hdGNoLmdyb3VwKDEpLnN0cmlwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXJfY29udGVudCUyMCUzRCUyMG1hdGNoLmdyb3VwKDIpLnN0cmlwKCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGxlbih0aGlua19jb250ZW50KSUyMCUzRSUyMDIwJTIwYW5kJTIwbGVuKGFuc3dlcl9jb250ZW50KSUyMCUzRSUyMDAlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZCgxLjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZHMuYXBwZW5kKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDAuNSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMjAlMjAlMjMlMjBQYXJ0aWFsJTIwcmV3YXJkJTIwZm9yJTIwY29ycmVjdCUyMGZvcm1hdCUyMGJ1dCUyMGxpbWl0ZWQlMjBjb250ZW50JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZHMuYXBwZW5kKDAuMCklMjAlMjAlMjMlMjBObyUyMHJld2FyZCUyMGZvciUyMGluY29ycmVjdCUyMGZvcm1hdCUwQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJld2FyZHM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">format_reward</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Reward completions that follow the desired format&quot;&quot;&quot;</span>
    <span class="hljs-comment"># Example: Check if the completion follows a think-then-answer format</span>
    pattern = <span class="hljs-string">r&quot;&lt;think&gt;(.*?)&lt;/think&gt;\\s*&lt;answer&gt;(.*?)&lt;/answer&gt;&quot;</span>

    rewards = []
    <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions:
        <span class="hljs-keyword">match</span> = re.search(pattern, completion, re.DOTALL)
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span>:
            <span class="hljs-comment"># Check if there&#x27;s substantial content in both sections</span>
            think_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">1</span>).strip()
            answer_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">2</span>).strip()

            <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(think_content) &gt; <span class="hljs-number">20</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(answer_content) &gt; <span class="hljs-number">0</span>:
                rewards.append(<span class="hljs-number">1.0</span>)
            <span class="hljs-keyword">else</span>:
                rewards.append(
                    <span class="hljs-number">0.5</span>
                )  <span class="hljs-comment"># Partial reward for correct format but limited content</span>
        <span class="hljs-keyword">else</span>:
            rewards.append(<span class="hljs-number">0.0</span>)  <span class="hljs-comment"># No reward for incorrect format</span>

    <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),ne=new j({props:{title:"That’s it!",local:"thats-it",headingTag:"h2"}}),re=new Fl({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter12/4.mdx"}}),{c(){u=i("meta"),ye=a(),Me=i("p"),we=a(),o(f.$$.fragment),ce=a(),o(C.$$.fragment),de=a(),I=i("p"),I.textContent=rl,Ue=a(),b=i("p"),b.textContent=pl,Je=a(),h=i("blockquote"),h.innerHTML=Ml,Te=a(),g=i("p"),g.textContent=ol,je=a(),G=i("ul"),G.innerHTML=ml,ue=a(),B=i("p"),B.textContent=yl,he=a(),Z=i("ul"),Z.innerHTML=wl,fe=a(),$=i("p"),$.textContent=cl,Ce=a(),o(R.$$.fragment),Ie=a(),o(k.$$.fragment),be=a(),o(A.$$.fragment),ge=a(),v=i("p"),v.textContent=dl,Ge=a(),o(x.$$.fragment),Be=a(),_=i("p"),_.textContent=Ul,Ze=a(),o(X.$$.fragment),$e=a(),o(E.$$.fragment),Re=a(),W=i("p"),W.innerHTML=Jl,ke=a(),o(F.$$.fragment),Ae=a(),Y=i("p"),Y.innerHTML=Tl,ve=a(),S=i("ul"),S.innerHTML=jl,xe=a(),z=i("p"),z.textContent=ul,_e=a(),o(Q.$$.fragment),Xe=a(),V=i("ol"),V.innerHTML=hl,Ee=a(),o(N.$$.fragment),We=a(),H=i("p"),H.textContent=fl,Fe=a(),o(L.$$.fragment),Ye=a(),q=i("p"),q.textContent=Cl,Se=a(),o(P.$$.fragment),ze=a(),D=i("p"),D.textContent=Il,Qe=a(),U=i("iframe"),Ve=a(),o(K.$$.fragment),Ne=a(),O=i("p"),O.textContent=gl,He=a(),o(ee.$$.fragment),Le=a(),J=i("iframe"),qe=a(),o(le.$$.fragment),Pe=a(),te=i("p"),te.textContent=Bl,De=a(),o(se.$$.fragment),Ke=a(),T=i("iframe"),Oe=a(),ae=i("p"),ae.textContent=$l,el=a(),o(ne.$$.fragment),ll=a(),ie=i("p"),ie.textContent=Rl,tl=a(),o(re.$$.fragment),sl=a(),oe=i("p"),this.h()},l(e){const l=Xl("svelte-u9bgzb",document.head);u=r(l,"META",{name:!0,content:!0}),l.forEach(t),ye=n(e),Me=r(e,"P",{}),me(Me).forEach(t),we=n(e),m(f.$$.fragment,e),ce=n(e),m(C.$$.fragment,e),de=n(e),I=r(e,"P",{"data-svelte-h":!0}),p(I)!=="svelte-v9iq5e"&&(I.textContent=rl),Ue=n(e),b=r(e,"P",{"data-svelte-h":!0}),p(b)!=="svelte-iqdzey"&&(b.textContent=pl),Je=n(e),h=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),p(h)!=="svelte-1a2kqv0"&&(h.innerHTML=Ml),Te=n(e),g=r(e,"P",{"data-svelte-h":!0}),p(g)!=="svelte-14hk95q"&&(g.textContent=ol),je=n(e),G=r(e,"UL",{"data-svelte-h":!0}),p(G)!=="svelte-1vd6wz6"&&(G.innerHTML=ml),ue=n(e),B=r(e,"P",{"data-svelte-h":!0}),p(B)!=="svelte-pzoe2y"&&(B.textContent=yl),he=n(e),Z=r(e,"UL",{"data-svelte-h":!0}),p(Z)!=="svelte-1hif2as"&&(Z.innerHTML=wl),fe=n(e),$=r(e,"P",{"data-svelte-h":!0}),p($)!=="svelte-ac9u4w"&&($.textContent=cl),Ce=n(e),m(R.$$.fragment,e),Ie=n(e),m(k.$$.fragment,e),be=n(e),m(A.$$.fragment,e),ge=n(e),v=r(e,"P",{"data-svelte-h":!0}),p(v)!=="svelte-1pnmwnq"&&(v.textContent=dl),Ge=n(e),m(x.$$.fragment,e),Be=n(e),_=r(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-ths948"&&(_.textContent=Ul),Ze=n(e),m(X.$$.fragment,e),$e=n(e),m(E.$$.fragment,e),Re=n(e),W=r(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-1uukcnx"&&(W.innerHTML=Jl),ke=n(e),m(F.$$.fragment,e),Ae=n(e),Y=r(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-1tdjaxe"&&(Y.innerHTML=Tl),ve=n(e),S=r(e,"UL",{"data-svelte-h":!0}),p(S)!=="svelte-1b1t69f"&&(S.innerHTML=jl),xe=n(e),z=r(e,"P",{"data-svelte-h":!0}),p(z)!=="svelte-7howml"&&(z.textContent=ul),_e=n(e),m(Q.$$.fragment,e),Xe=n(e),V=r(e,"OL",{"data-svelte-h":!0}),p(V)!=="svelte-130a5z3"&&(V.innerHTML=hl),Ee=n(e),m(N.$$.fragment,e),We=n(e),H=r(e,"P",{"data-svelte-h":!0}),p(H)!=="svelte-5lvuy8"&&(H.textContent=fl),Fe=n(e),m(L.$$.fragment,e),Ye=n(e),q=r(e,"P",{"data-svelte-h":!0}),p(q)!=="svelte-jhf03w"&&(q.textContent=Cl),Se=n(e),m(P.$$.fragment,e),ze=n(e),D=r(e,"P",{"data-svelte-h":!0}),p(D)!=="svelte-jlmna3"&&(D.textContent=Il),Qe=n(e),U=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),me(U).forEach(t),Ve=n(e),m(K.$$.fragment,e),Ne=n(e),O=r(e,"P",{"data-svelte-h":!0}),p(O)!=="svelte-mrpax"&&(O.textContent=gl),He=n(e),m(ee.$$.fragment,e),Le=n(e),J=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),me(J).forEach(t),qe=n(e),m(le.$$.fragment,e),Pe=n(e),te=r(e,"P",{"data-svelte-h":!0}),p(te)!=="svelte-18k4wdv"&&(te.textContent=Bl),De=n(e),m(se.$$.fragment,e),Ke=n(e),T=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),me(T).forEach(t),Oe=n(e),ae=r(e,"P",{"data-svelte-h":!0}),p(ae)!=="svelte-fzl1fe"&&(ae.textContent=$l),el=n(e),m(ne.$$.fragment,e),ll=n(e),ie=r(e,"P",{"data-svelte-h":!0}),p(ie)!=="svelte-chd5s6"&&(ie.textContent=Rl),tl=n(e),m(re.$$.fragment,e),sl=n(e),oe=r(e,"P",{}),me(oe).forEach(t),this.h()},h(){M(u,"name","hf:doc:metadata"),M(u,"content",Sl),M(h,"class","tip"),nl(U.src,bl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_length.py&embed=true&show-chrome=false")||M(U,"src",bl),M(U,"title","Marimo Notebook"),M(U,"width","100%"),M(U,"height","800px"),M(U,"frameborder","0"),M(U,"allow","clipboard-write"),nl(J.src,Gl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_math.py&embed=true&show-chrome=false")||M(J,"src",Gl),M(J,"title","Marimo Notebook"),M(J,"width","100%"),M(J,"height","800px"),M(J,"frameborder","0"),M(J,"allow","clipboard-write"),nl(T.src,Zl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_format.py&embed=true&show-chrome=false")||M(T,"src",Zl),M(T,"title","Marimo Notebook"),M(T,"width","100%"),M(T,"height","800px"),M(T,"frameborder","0"),M(T,"allow","clipboard-write")},m(e,l){El(document.head,u),s(e,ye,l),s(e,Me,l),s(e,we,l),y(f,e,l),s(e,ce,l),y(C,e,l),s(e,de,l),s(e,I,l),s(e,Ue,l),s(e,b,l),s(e,Je,l),s(e,h,l),s(e,Te,l),s(e,g,l),s(e,je,l),s(e,G,l),s(e,ue,l),s(e,B,l),s(e,he,l),s(e,Z,l),s(e,fe,l),s(e,$,l),s(e,Ce,l),y(R,e,l),s(e,Ie,l),y(k,e,l),s(e,be,l),y(A,e,l),s(e,ge,l),s(e,v,l),s(e,Ge,l),y(x,e,l),s(e,Be,l),s(e,_,l),s(e,Ze,l),y(X,e,l),s(e,$e,l),y(E,e,l),s(e,Re,l),s(e,W,l),s(e,ke,l),y(F,e,l),s(e,Ae,l),s(e,Y,l),s(e,ve,l),s(e,S,l),s(e,xe,l),s(e,z,l),s(e,_e,l),y(Q,e,l),s(e,Xe,l),s(e,V,l),s(e,Ee,l),y(N,e,l),s(e,We,l),s(e,H,l),s(e,Fe,l),y(L,e,l),s(e,Ye,l),s(e,q,l),s(e,Se,l),y(P,e,l),s(e,ze,l),s(e,D,l),s(e,Qe,l),s(e,U,l),s(e,Ve,l),y(K,e,l),s(e,Ne,l),s(e,O,l),s(e,He,l),y(ee,e,l),s(e,Le,l),s(e,J,l),s(e,qe,l),y(le,e,l),s(e,Pe,l),s(e,te,l),s(e,De,l),y(se,e,l),s(e,Ke,l),s(e,T,l),s(e,Oe,l),s(e,ae,l),s(e,el,l),y(ne,e,l),s(e,ll,l),s(e,ie,l),s(e,tl,l),y(re,e,l),s(e,sl,l),s(e,oe,l),al=!0},p:Al,i(e){al||(w(f.$$.fragment,e),w(C.$$.fragment,e),w(R.$$.fragment,e),w(k.$$.fragment,e),w(A.$$.fragment,e),w(x.$$.fragment,e),w(X.$$.fragment,e),w(E.$$.fragment,e),w(F.$$.fragment,e),w(Q.$$.fragment,e),w(N.$$.fragment,e),w(L.$$.fragment,e),w(P.$$.fragment,e),w(K.$$.fragment,e),w(ee.$$.fragment,e),w(le.$$.fragment,e),w(se.$$.fragment,e),w(ne.$$.fragment,e),w(re.$$.fragment,e),al=!0)},o(e){c(f.$$.fragment,e),c(C.$$.fragment,e),c(R.$$.fragment,e),c(k.$$.fragment,e),c(A.$$.fragment,e),c(x.$$.fragment,e),c(X.$$.fragment,e),c(E.$$.fragment,e),c(F.$$.fragment,e),c(Q.$$.fragment,e),c(N.$$.fragment,e),c(L.$$.fragment,e),c(P.$$.fragment,e),c(K.$$.fragment,e),c(ee.$$.fragment,e),c(le.$$.fragment,e),c(se.$$.fragment,e),c(ne.$$.fragment,e),c(re.$$.fragment,e),al=!1},d(e){e&&(t(ye),t(Me),t(we),t(ce),t(de),t(I),t(Ue),t(b),t(Je),t(h),t(Te),t(g),t(je),t(G),t(ue),t(B),t(he),t(Z),t(fe),t($),t(Ce),t(Ie),t(be),t(ge),t(v),t(Ge),t(Be),t(_),t(Ze),t($e),t(Re),t(W),t(ke),t(Ae),t(Y),t(ve),t(S),t(xe),t(z),t(_e),t(Xe),t(V),t(Ee),t(We),t(H),t(Fe),t(Ye),t(q),t(Se),t(ze),t(D),t(Qe),t(U),t(Ve),t(Ne),t(O),t(He),t(Le),t(J),t(qe),t(Pe),t(te),t(De),t(Ke),t(T),t(Oe),t(ae),t(el),t(ll),t(ie),t(tl),t(sl),t(oe)),t(u),d(f,e),d(C,e),d(R,e),d(k,e),d(A,e),d(x,e),d(X,e),d(E,e),d(F,e),d(Q,e),d(N,e),d(L,e),d(P,e),d(K,e),d(ee,e),d(le,e),d(se,e),d(ne,e),d(re,e)}}}const Sl='{"title":"Implementing GRPO in TRL","local":"implementing-grpo-in-trl","sections":[{"title":"Key Components","local":"key-components","sections":[{"title":"1. Dataset Format","local":"1-dataset-format","sections":[],"depth":3},{"title":"2. Reward Function","local":"2-reward-function","sections":[],"depth":3},{"title":"3. Training Configuration","local":"3-training-configuration","sections":[],"depth":3}],"depth":2},{"title":"Tips for Success","local":"tips-for-success","sections":[],"depth":2},{"title":"Reward Function Design","local":"reward-function-design","sections":[{"title":"1. Length-Based Rewards","local":"1-length-based-rewards","sections":[],"depth":3}],"depth":2},{"title":"2. Rule-Based Rewards for Verifiable Tasks","local":"2-rule-based-rewards-for-verifiable-tasks","sections":[],"depth":2},{"title":"3. Format-Based Rewards","local":"3-format-based-rewards","sections":[],"depth":2},{"title":"That’s it!","local":"thats-it","sections":[],"depth":2}],"depth":1}';function zl(il){return vl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ll extends xl{constructor(u){super(),_l(this,u,zl,Yl,kl,{})}}export{Ll as component};