course/pr_1107/en/_app/immutable/nodes/32.008dcb60.js

import{s as kl,a as sl,n as Rl,o as Al}from"../chunks/scheduler.0835143d.js";import{S as $l,i as vl,e as i,s as a,c as o,h as _l,a as r,d as t,b as n,f as oe,g as y,j as M,k as p,l as xl,m as s,n as m,t as w,o as c,p as d}from"../chunks/index.3d7efe79.js";import{C as re}from"../chunks/CodeBlock.116ed840.js";import{H as j,E as Xl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f20191d3.js";function El(al){let u,ye,Me,me,f,we,C,nl="In this page, we’ll learn how to implement Group Relative Policy Optimization (GRPO) using the Transformer Reinforcement Learning (TRL) library. We’ll focus on practical implementation with minimal code.",ce,I,il="We’ll explore the core concepts of GRPO as they are embodied in TRL’s GRPOTrainer, using snippets from the official TRL documentation to guide us.",de,h,rl='<p>This chapter is aimed at TRL beginners. If you are already familiar with TRL, you might want to also check out the <a href="https://github.com/huggingface/open-r1/blob/main/src/open_r1/grpo.py" rel="nofollow">Open R1 implementation</a> of GRPO.</p>',Ue,b,Ml="First, let’s remind ourselves of some of the important concepts of GRPO algorithm:",Je,g,pl="<li>Group Formation: The model generates multiple completions for each prompt.</li> <li>Preference Learning: The model learns from a reward function that compares groups of completions.</li> <li>Training Configuration: The model uses a configuration to control the training process.</li>",Te,G,ol="What do we need to do to implement GRPO?",je,B,yl="<li>Define a dataset of prompts.</li> <li>Define a reward function that takes a list of completions and returns a list of rewards.</li> <li>Configure the training process with a GRPOConfig.</li> <li>Train the model using the GRPOTrainer.</li>",ue,Z,ml="Here’s a minimal example to get started with GRPO training:",he,k,fe,R,Ce,A,Ie,$,wl="Your dataset should contain prompts that the model will respond to. The GRPO trainer will generate multiple completions for each prompt and use the reward function to compare them.",be,v,ge,_,cl="The reward function is crucial - it determines how the model learns. Here are two practical examples:",Ge,x,Be,X,Ze,E,dl="Key parameters to consider in <code>GRPOConfig</code>:",ke,W,Re,F,Ul="The <code>num_generation</code> parameter is particularly important for GRPO as it defines the group size - how many different completions the model will generate for each prompt. This is a key differentiator from other RL methods:",Ae,Y,Jl="<li>Too small (e.g., 2-3): May not provide enough diversity for meaningful comparisons</li> <li>Recommended (4-16): Provides good balance between diversity and computational efficiency</li> <li>Larger values: May improve learning but significantly increases computational cost</li>",$e,S,Tl="The group size should be chosen based on your computational resources and the complexity of your task. For simple tasks, smaller groups (4-8) may be sufficient, while more complex reasoning tasks might benefit from larger groups (8-16).",ve,Q,_e,V,jl="<li><strong>Memory Management</strong>: Adjust <code>per_device_train_batch_size</code> and <code>gradient_accumulation_steps</code> based on your GPU memory.</li> <li><strong>Speed</strong>: Enable <code>use_vllm=True</code> for faster generation if your model is supported.</li> <li><strong>Monitoring</strong>: Watch the logged metrics during training:<ul><li><code>reward</code>: Average reward across completions</li> <li><code>reward_std</code>: Standard deviation within reward groups</li> <li><code>kl</code>: KL divergence from reference model</li></ul></li>",xe,z,Xe,N,ul="The DeepSeek R1 paper demonstrates several effective approaches to reward function design that you can adapt for your own GRPO implementation:",Ee,H,We,L,hl="One of the easiest rewards to implement is a length-based reward. You can reward longer completions:",Fe,q,Ye,P,fl="This reward function penalizes completions that are too short or too long, encouraging the model to generate completions that are close to the ideal length of 20 tokens.",Se,U,Cl,Qe,D,Ve,K,Il="For tasks with objectively correct answers (like mathematics or coding), you can implement rule-based reward functions:",ze,O,Ne,J,bl,He,ee,Le,le,gl="You can also reward proper formatting, which was important in the DeepSeek R1 training:",qe,te,Pe,T,Gl,De,se,Bl="These examples demonstrate how you can implement reward functions inspired by the DeepSeek R1 training process, focusing on correctness, formatting, and combined signals.",Ke,ae,Oe,ne,Zl="In the next section, you will follow an exercise to implement GRPO in TRL.",el,ie,ll,pe,tl;return f=new j({props:{title:"Implementing GRPO in TRL",local:"implementing-grpo-in-trl",headingTag:"h1"}}),k=new re({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTJDJTIwR1JQT0NvbmZpZyUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQSUwQSUyMyUyMDEuJTIwTG9hZCUyMHlvdXIlMjBkYXRhc2V0JTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ5b3VyX2RhdGFzZXQlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQSUwQSUyMyUyMDIuJTIwRGVmaW5lJTIwYSUyMHNpbXBsZSUyMHJld2FyZCUyMGZ1bmN0aW9uJTBBZGVmJTIwcmV3YXJkX2Z1bmMoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjJFeGFtcGxlJTNBJTIwUmV3YXJkJTIwbG9uZ2VyJTIwY29tcGxldGlvbnMlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmbG9hdChsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMEElMEElMjMlMjAzLiUyMENvbmZpZ3VyZSUyMHRyYWluaW5nJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm91dHB1dCUyMiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QzJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDIlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEMTAlMkMlMEEpJTBBJTBBJTIzJTIwNC4lMjBJbml0aWFsaXplJTIwYW5kJTIwdHJhaW4lMEF0cmFpbmVyJTIwJTNEJTIwR1JQT1RyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJ5b3VyX21vZGVsJTIyJTJDJTIwJTIwJTIzJTIwZS5nLiUyMCUyMlF3ZW4lMkZRd2VuMi0wLjVCLUluc3RydWN0JTIyJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRHJld2FyZF9mdW5jJTJDJTBBKSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer, GRPOConfig
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># 1. Load your dataset</span>
dataset = load_dataset(<span class="hljs-string">&quot;your_dataset&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)


<span class="hljs-comment"># 2. Define a simple reward function</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Example: Reward longer completions&quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]


<span class="hljs-comment"># 3. Configure training</span>
training_args = GRPOConfig(
    output_dir=<span class="hljs-string">&quot;output&quot;</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    per_device_train_batch_size=<span class="hljs-number">4</span>,
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    logging_steps=<span class="hljs-number">10</span>,
)

<span class="hljs-comment"># 4. Initialize and train</span>
trainer = GRPOTrainer(
    model=<span class="hljs-string">&quot;your_model&quot;</span>,  <span class="hljs-comment"># e.g. &quot;Qwen/Qwen2-0.5B-Instruct&quot;</span>
    args=training_args,
    train_dataset=dataset,
    reward_funcs=reward_func,
)
trainer.train()`,wrap:!1}}),R=new j({props:{title:"Key Components",local:"key-components",headingTag:"h2"}}),A=new j({props:{title:"1. Dataset Format",local:"1-dataset-format",headingTag:"h3"}}),v=new j({props:{title:"2. Reward Function",local:"2-reward-function",headingTag:"h3"}}),x=new re({props:{code:"JTIzJTIwRXhhbXBsZSUyMDElM0ElMjBSZXdhcmQlMjBiYXNlZCUyMG9uJTIwY29tcGxldGlvbiUyMGxlbmd0aCUwQWRlZiUyMHJld2FyZF9sZW5ndGgoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmbG9hdChsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMEElMEElMjMlMjBFeGFtcGxlJTIwMiUzQSUyMFJld2FyZCUyMGJhc2VkJTIwb24lMjBtYXRjaGluZyUyMGElMjBwYXR0ZXJuJTBBaW1wb3J0JTIwcmUlMEElMEElMEFkZWYlMjByZXdhcmRfZm9ybWF0KGNvbXBsZXRpb25zJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwcGF0dGVybiUyMCUzRCUyMHIlMjIlNUUlM0N0aGluayUzRS4qJTNGJTNDJTJGdGhpbmslM0UlM0NhbnN3ZXIlM0UuKiUzRiUzQyUyRmFuc3dlciUzRSUyNCUyMiUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QjEuMCUyMGlmJTIwcmUubWF0Y2gocGF0dGVybiUyQyUyMGMpJTIwZWxzZSUyMDAuMCUyMGZvciUyMGMlMjBpbiUyMGNvbXBsZXRpb25zJTVE",highlighted:`<span class="hljs-comment"># Example 1: Reward based on completion length</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_length</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]


<span class="hljs-comment"># Example 2: Reward based on matching a pattern</span>
<span class="hljs-keyword">import</span> re


<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_format</span>(<span class="hljs-params">completions, **kwargs</span>):
    pattern = <span class="hljs-string">r&quot;^&lt;think&gt;.*?&lt;/think&gt;&lt;answer&gt;.*?&lt;/answer&gt;$&quot;</span>
    <span class="hljs-keyword">return</span> [<span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> re.<span class="hljs-keyword">match</span>(pattern, c) <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),X=new j({props:{title:"3. Training Configuration",local:"3-training-configuration",headingTag:"h3"}}),W=new re({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwJTIzJTIwRXNzZW50aWFsJTIwcGFyYW1ldGVycyUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJvdXRwdXQlMjIlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMyUyQyUwQSUyMCUyMCUyMCUyMG51bV9nZW5lcmF0aW9uJTNENCUyQyUyMCUyMCUyMyUyME51bWJlciUyMG9mJTIwY29tcGxldGlvbnMlMjB0byUyMGdlbmVyYXRlJTIwZm9yJTIwZWFjaCUyMHByb21wdCUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDQlMkMlMjAlMjAlMjMlMjBXZSUyMHdhbnQlMjB0byUyMGdldCUyMGFsbCUyMGdlbmVyYXRpb25zJTIwaW4lMjBvbmUlMjBkZXZpY2UlMjBiYXRjaCUwQSUyMCUyMCUyMCUyMCUyMyUyME9wdGlvbmFsJTIwYnV0JTIwdXNlZnVsJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNEMiUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QxZS01JTJDJTBBJTIwJTIwJTIwJTIwbG9nZ2luZ19zdGVwcyUzRDEwJTJDJTBBJTIwJTIwJTIwJTIwJTIzJTIwR1JQTyUyMHNwZWNpZmljJTIwKG9wdGlvbmFsKSUwQSUyMCUyMCUyMCUyMHVzZV92bGxtJTNEVHJ1ZSUyQyUyMCUyMCUyMyUyMFNwZWVkJTIwdXAlMjBnZW5lcmF0aW9uJTBBKQ==",highlighted:`training_args = GRPOConfig(
    <span class="hljs-comment"># Essential parameters</span>
    output_dir=<span class="hljs-string">&quot;output&quot;</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    num_generation=<span class="hljs-number">4</span>,  <span class="hljs-comment"># Number of completions to generate for each prompt</span>
    per_device_train_batch_size=<span class="hljs-number">4</span>,  <span class="hljs-comment"># We want to get all generations in one device batch</span>
    <span class="hljs-comment"># Optional but useful</span>
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    learning_rate=<span class="hljs-number">1e-5</span>,
    logging_steps=<span class="hljs-number">10</span>,
    <span class="hljs-comment"># GRPO specific (optional)</span>
    use_vllm=<span class="hljs-literal">True</span>,  <span class="hljs-comment"># Speed up generation</span>
)`,wrap:!1}}),Q=new j({props:{title:"Tips for Success",local:"tips-for-success",headingTag:"h2"}}),z=new j({props:{title:"Reward Function Design",local:"reward-function-design",headingTag:"h2"}}),H=new j({props:{title:"1. Length-Based Rewards",local:"1-length-based-rewards",headingTag:"h3"}}),q=new re({props:{code:"ZGVmJTIwcmV3YXJkX2xlbihjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMGlkZWFsX2xlbmd0aCUyMCUzRCUyMDIwJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTVCLWFicyhpZGVhbF9sZW5ndGglMjAtJTIwbGVuKGNvbXBsZXRpb24pKSUyMGZvciUyMGNvbXBsZXRpb24lMjBpbiUyMGNvbXBsZXRpb25zJTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_len</span>(<span class="hljs-params">completions, **kwargs</span>):
    ideal_length = <span class="hljs-number">20</span>
    <span class="hljs-keyword">return</span> [-<span class="hljs-built_in">abs</span>(ideal_length - <span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),D=new j({props:{title:"2. Rule-Based Rewards for Verifiable Tasks",local:"2-rule-based-rewards-for-verifiable-tasks",headingTag:"h2"}}),O=new re({props:{code:"ZGVmJTIwcHJvYmxlbV9yZXdhcmQoY29tcGxldGlvbnMlMkMlMjBhbnN3ZXJzJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyUmV3YXJkJTIwZnVuY3Rpb24lMjBmb3IlMjBtYXRoJTIwcHJvYmxlbXMlMjB3aXRoJTIwdmVyaWZpYWJsZSUyMGFuc3dlcnMlMEElMjAlMjAlMjAlMjBjb21wbGV0aW9ucyUzQSUyMGxpc3QlMjBvZiUyMGNvbXBsZXRpb25zJTIwdG8lMjBldmFsdWF0ZSUwQSUyMCUyMCUyMCUyMGFuc3dlcnMlM0ElMjBsaXN0JTIwb2YlMjBhbnN3ZXJzJTIwdG8lMjB0aGUlMjBwcm9ibGVtcyUyMGZyb20lMjB0aGUlMjBkYXRhc2V0JTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTBBJTIwJTIwJTIwJTIwcmV3YXJkcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGNvbXBsZXRpb24lMkMlMjBjb3JyZWN0X2Fuc3dlciUyMGluJTIwemlwKGNvbXBsZXRpb25zJTJDJTIwYW5zd2VycyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBFeHRyYWN0JTIwdGhlJTIwYW5zd2VyJTIwZnJvbSUyMHRoZSUyMGNvbXBsZXRpb24lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBUaGlzJTIwaXMlMjBhJTIwc2ltcGxpZmllZCUyMGV4YW1wbGUlMjAtJTIweW91J2QlMjBuZWVkJTIwcHJvcGVyJTIwcGFyc2luZyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFuc3dlciUyMCUzRCUyMGV4dHJhY3RfZmluYWxfYW5zd2VyKGNvbXBsZXRpb24pJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwQmluYXJ5JTIwcmV3YXJkJTNBJTIwMSUyMGZvciUyMGNvcnJlY3QlMkMlMjAwJTIwZm9yJTIwaW5jb3JyZWN0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV3YXJkJTIwJTNEJTIwMS4wJTIwaWYlMjBhbnN3ZXIlMjAlM0QlM0QlMjBjb3JyZWN0X2Fuc3dlciUyMGVsc2UlMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZChyZXdhcmQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXhjZXB0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwSWYlMjB3ZSUyMGNhbid0JTIwcGFyc2UlMjBhbiUyMGFuc3dlciUyQyUyMGdpdmUlMjBhJTIwbG93JTIwcmV3YXJkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV3YXJkcy5hcHBlbmQoMC4wKSUwQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJld2FyZHM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">problem_reward</span>(<span class="hljs-params">completions, answers, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Reward function for math problems with verifiable answers
    completions: list of completions to evaluate
    answers: list of answers to the problems from the dataset
    &quot;&quot;&quot;</span>

    rewards = []
    <span class="hljs-keyword">for</span> completion, correct_answer <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(completions, answers):
        <span class="hljs-comment"># Extract the answer from the completion</span>
        <span class="hljs-keyword">try</span>:
            <span class="hljs-comment"># This is a simplified example - you&#x27;d need proper parsing</span>
            answer = extract_final_answer(completion)
            <span class="hljs-comment"># Binary reward: 1 for correct, 0 for incorrect</span>
            reward = <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> answer == correct_answer <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span>
            rewards.append(reward)
        <span class="hljs-keyword">except</span>:
            <span class="hljs-comment"># If we can&#x27;t parse an answer, give a low reward</span>
            rewards.append(<span class="hljs-number">0.0</span>)

    <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),ee=new j({props:{title:"3. Format-Based Rewards",local:"3-format-based-rewards",headingTag:"h2"}}),te=new re({props:{code:"ZGVmJTIwZm9ybWF0X3Jld2FyZChjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMlJld2FyZCUyMGNvbXBsZXRpb25zJTIwdGhhdCUyMGZvbGxvdyUyMHRoZSUyMGRlc2lyZWQlMjBmb3JtYXQlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjAlMjMlMjBFeGFtcGxlJTNBJTIwQ2hlY2slMjBpZiUyMHRoZSUyMGNvbXBsZXRpb24lMjBmb2xsb3dzJTIwYSUyMHRoaW5rLXRoZW4tYW5zd2VyJTIwZm9ybWF0JTBBJTIwJTIwJTIwJTIwcGF0dGVybiUyMCUzRCUyMHIlMjIlM0N0aGluayUzRSguKiUzRiklM0MlMkZ0aGluayUzRSU1Q3MqJTNDYW5zd2VyJTNFKC4qJTNGKSUzQyUyRmFuc3dlciUzRSUyMiUwQSUwQSUyMCUyMCUyMCUyMHJld2FyZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBjb21wbGV0aW9uJTIwaW4lMjBjb21wbGV0aW9ucyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hdGNoJTIwJTNEJTIwcmUuc2VhcmNoKHBhdHRlcm4lMkMlMjBjb21wbGV0aW9uJTJDJTIwcmUuRE9UQUxMKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwbWF0Y2glM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBDaGVjayUyMGlmJTIwdGhlcmUncyUyMHN1YnN0YW50aWFsJTIwY29udGVudCUyMGluJTIwYm90aCUyMHNlY3Rpb25zJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdGhpbmtfY29udGVudCUyMCUzRCUyMG1hdGNoLmdyb3VwKDEpLnN0cmlwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXJfY29udGVudCUyMCUzRCUyMG1hdGNoLmdyb3VwKDIpLnN0cmlwKCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGxlbih0aGlua19jb250ZW50KSUyMCUzRSUyMDIwJTIwYW5kJTIwbGVuKGFuc3dlcl9jb250ZW50KSUyMCUzRSUyMDAlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZCgxLjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZHMuYXBwZW5kKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDAuNSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMjAlMjAlMjMlMjBQYXJ0aWFsJTIwcmV3YXJkJTIwZm9yJTIwY29ycmVjdCUyMGZvcm1hdCUyMGJ1dCUyMGxpbWl0ZWQlMjBjb250ZW50JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZHMuYXBwZW5kKDAuMCklMjAlMjAlMjMlMjBObyUyMHJld2FyZCUyMGZvciUyMGluY29ycmVjdCUyMGZvcm1hdCUwQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJld2FyZHM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">format_reward</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Reward completions that follow the desired format&quot;&quot;&quot;</span>
    <span class="hljs-comment"># Example: Check if the completion follows a think-then-answer format</span>
    pattern = <span class="hljs-string">r&quot;&lt;think&gt;(.*?)&lt;/think&gt;\\s*&lt;answer&gt;(.*?)&lt;/answer&gt;&quot;</span>

    rewards = []
    <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions:
        <span class="hljs-keyword">match</span> = re.search(pattern, completion, re.DOTALL)
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span>:
            <span class="hljs-comment"># Check if there&#x27;s substantial content in both sections</span>
            think_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">1</span>).strip()
            answer_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">2</span>).strip()

            <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(think_content) &gt; <span class="hljs-number">20</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(answer_content) &gt; <span class="hljs-number">0</span>:
                rewards.append(<span class="hljs-number">1.0</span>)
            <span class="hljs-keyword">else</span>:
                rewards.append(
                    <span class="hljs-number">0.5</span>
                )  <span class="hljs-comment"># Partial reward for correct format but limited content</span>
        <span class="hljs-keyword">else</span>:
            rewards.append(<span class="hljs-number">0.0</span>)  <span class="hljs-comment"># No reward for incorrect format</span>

    <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),ae=new j({props:{title:"That’s it!",local:"thats-it",headingTag:"h2"}}),ie=new Xl({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter12/4.mdx"}}),{c(){u=i("meta"),ye=a(),Me=i("p"),me=a(),o(f.$$.fragment),we=a(),C=i("p"),C.textContent=nl,ce=a(),I=i("p"),I.textContent=il,de=a(),h=i("blockquote"),h.innerHTML=rl,Ue=a(),b=i("p"),b.textContent=Ml,Je=a(),g=i("ul"),g.innerHTML=pl,Te=a(),G=i("p"),G.textContent=ol,je=a(),B=i("ul"),B.innerHTML=yl,ue=a(),Z=i("p"),Z.textContent=ml,he=a(),o(k.$$.fragment),fe=a(),o(R.$$.fragment),Ce=a(),o(A.$$.fragment),Ie=a(),$=i("p"),$.textContent=wl,be=a(),o(v.$$.fragment),ge=a(),_=i("p"),_.textContent=cl,Ge=a(),o(x.$$.fragment),Be=a(),o(X.$$.fragment),Ze=a(),E=i("p"),E.innerHTML=dl,ke=a(),o(W.$$.fragment),Re=a(),F=i("p"),F.innerHTML=Ul,Ae=a(),Y=i("ul"),Y.innerHTML=Jl,$e=a(),S=i("p"),S.textContent=Tl,ve=a(),o(Q.$$.fragment),_e=a(),V=i("ol"),V.innerHTML=jl,xe=a(),o(z.$$.fragment),Xe=a(),N=i("p"),N.textContent=ul,Ee=a(),o(H.$$.fragment),We=a(),L=i("p"),L.textContent=hl,Fe=a(),o(q.$$.fragment),Ye=a(),P=i("p"),P.textContent=fl,Se=a(),U=i("iframe"),Qe=a(),o(D.$$.fragment),Ve=a(),K=i("p"),K.textContent=Il,ze=a(),o(O.$$.fragment),Ne=a(),J=i("iframe"),He=a(),o(ee.$$.fragment),Le=a(),le=i("p"),le.textContent=gl,qe=a(),o(te.$$.fragment),Pe=a(),T=i("iframe"),De=a(),se=i("p"),se.textContent=Bl,Ke=a(),o(ae.$$.fragment),Oe=a(),ne=i("p"),ne.textContent=Zl,el=a(),o(ie.$$.fragment),ll=a(),pe=i("p"),this.h()},l(e){const l=_l("svelte-u9bgzb",document.head);u=r(l,"META",{name:!0,content:!0}),l.forEach(t),ye=n(e),Me=r(e,"P",{}),oe(Me).forEach(t),me=n(e),y(f.$$.fragment,e),we=n(e),C=r(e,"P",{"data-svelte-h":!0}),M(C)!=="svelte-v9iq5e"&&(C.textContent=nl),ce=n(e),I=r(e,"P",{"data-svelte-h":!0}),M(I)!=="svelte-iqdzey"&&(I.textContent=il),de=n(e),h=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),M(h)!=="svelte-1a2kqv0"&&(h.innerHTML=rl),Ue=n(e),b=r(e,"P",{"data-svelte-h":!0}),M(b)!=="svelte-14hk95q"&&(b.textContent=Ml),Je=n(e),g=r(e,"UL",{"data-svelte-h":!0}),M(g)!=="svelte-1vd6wz6"&&(g.innerHTML=pl),Te=n(e),G=r(e,"P",{"data-svelte-h":!0}),M(G)!=="svelte-pzoe2y"&&(G.textContent=ol),je=n(e),B=r(e,"UL",{"data-svelte-h":!0}),M(B)!=="svelte-1hif2as"&&(B.innerHTML=yl),ue=n(e),Z=r(e,"P",{"data-svelte-h":!0}),M(Z)!=="svelte-ac9u4w"&&(Z.textContent=ml),he=n(e),y(k.$$.fragment,e),fe=n(e),y(R.$$.fragment,e),Ce=n(e),y(A.$$.fragment,e),Ie=n(e),$=r(e,"P",{"data-svelte-h":!0}),M($)!=="svelte-1pnmwnq"&&($.textContent=wl),be=n(e),y(v.$$.fragment,e),ge=n(e),_=r(e,"P",{"data-svelte-h":!0}),M(_)!=="svelte-ths948"&&(_.textContent=cl),Ge=n(e),y(x.$$.fragment,e),Be=n(e),y(X.$$.fragment,e),Ze=n(e),E=r(e,"P",{"data-svelte-h":!0}),M(E)!=="svelte-1uukcnx"&&(E.innerHTML=dl),ke=n(e),y(W.$$.fragment,e),Re=n(e),F=r(e,"P",{"data-svelte-h":!0}),M(F)!=="svelte-1tdjaxe"&&(F.innerHTML=Ul),Ae=n(e),Y=r(e,"UL",{"data-svelte-h":!0}),M(Y)!=="svelte-1b1t69f"&&(Y.innerHTML=Jl),$e=n(e),S=r(e,"P",{"data-svelte-h":!0}),M(S)!=="svelte-7howml"&&(S.textContent=Tl),ve=n(e),y(Q.$$.fragment,e),_e=n(e),V=r(e,"OL",{"data-svelte-h":!0}),M(V)!=="svelte-4ukx5l"&&(V.innerHTML=jl),xe=n(e),y(z.$$.fragment,e),Xe=n(e),N=r(e,"P",{"data-svelte-h":!0}),M(N)!=="svelte-5lvuy8"&&(N.textContent=ul),Ee=n(e),y(H.$$.fragment,e),We=n(e),L=r(e,"P",{"data-svelte-h":!0}),M(L)!=="svelte-jhf03w"&&(L.textContent=hl),Fe=n(e),y(q.$$.fragment,e),Ye=n(e),P=r(e,"P",{"data-svelte-h":!0}),M(P)!=="svelte-jlmna3"&&(P.textContent=fl),Se=n(e),U=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),oe(U).forEach(t),Qe=n(e),y(D.$$.fragment,e),Ve=n(e),K=r(e,"P",{"data-svelte-h":!0}),M(K)!=="svelte-mrpax"&&(K.textContent=Il),ze=n(e),y(O.$$.fragment,e),Ne=n(e),J=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),oe(J).forEach(t),He=n(e),y(ee.$$.fragment,e),Le=n(e),le=r(e,"P",{"data-svelte-h":!0}),M(le)!=="svelte-18k4wdv"&&(le.textContent=gl),qe=n(e),y(te.$$.fragment,e),Pe=n(e),T=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),oe(T).forEach(t),De=n(e),se=r(e,"P",{"data-svelte-h":!0}),M(se)!=="svelte-fzl1fe"&&(se.textContent=Bl),Ke=n(e),y(ae.$$.fragment,e),Oe=n(e),ne=r(e,"P",{"data-svelte-h":!0}),M(ne)!=="svelte-chd5s6"&&(ne.textContent=Zl),el=n(e),y(ie.$$.fragment,e),ll=n(e),pe=r(e,"P",{}),oe(pe).forEach(t),this.h()},h(){p(u,"name","hf:doc:metadata"),p(u,"content",Wl),p(h,"class","tip"),sl(U.src,Cl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_length.py&embed=true&show-chrome=false")||p(U,"src",Cl),p(U,"title","Marimo Notebook"),p(U,"width","100%"),p(U,"height","800px"),p(U,"frameborder","0"),p(U,"allow","clipboard-write"),sl(J.src,bl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_math.py&embed=true&show-chrome=false")||p(J,"src",bl),p(J,"title","Marimo Notebook"),p(J,"width","100%"),p(J,"height","800px"),p(J,"frameborder","0"),p(J,"allow","clipboard-write"),sl(T.src,Gl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_format.py&embed=true&show-chrome=false")||p(T,"src",Gl),p(T,"title","Marimo Notebook"),p(T,"width","100%"),p(T,"height","800px"),p(T,"frameborder","0"),p(T,"allow","clipboard-write")},m(e,l){xl(document.head,u),s(e,ye,l),s(e,Me,l),s(e,me,l),m(f,e,l),s(e,we,l),s(e,C,l),s(e,ce,l),s(e,I,l),s(e,de,l),s(e,h,l),s(e,Ue,l),s(e,b,l),s(e,Je,l),s(e,g,l),s(e,Te,l),s(e,G,l),s(e,je,l),s(e,B,l),s(e,ue,l),s(e,Z,l),s(e,he,l),m(k,e,l),s(e,fe,l),m(R,e,l),s(e,Ce,l),m(A,e,l),s(e,Ie,l),s(e,$,l),s(e,be,l),m(v,e,l),s(e,ge,l),s(e,_,l),s(e,Ge,l),m(x,e,l),s(e,Be,l),m(X,e,l),s(e,Ze,l),s(e,E,l),s(e,ke,l),m(W,e,l),s(e,Re,l),s(e,F,l),s(e,Ae,l),s(e,Y,l),s(e,$e,l),s(e,S,l),s(e,ve,l),m(Q,e,l),s(e,_e,l),s(e,V,l),s(e,xe,l),m(z,e,l),s(e,Xe,l),s(e,N,l),s(e,Ee,l),m(H,e,l),s(e,We,l),s(e,L,l),s(e,Fe,l),m(q,e,l),s(e,Ye,l),s(e,P,l),s(e,Se,l),s(e,U,l),s(e,Qe,l),m(D,e,l),s(e,Ve,l),s(e,K,l),s(e,ze,l),m(O,e,l),s(e,Ne,l),s(e,J,l),s(e,He,l),m(ee,e,l),s(e,Le,l),s(e,le,l),s(e,qe,l),m(te,e,l),s(e,Pe,l),s(e,T,l),s(e,De,l),s(e,se,l),s(e,Ke,l),m(ae,e,l),s(e,Oe,l),s(e,ne,l),s(e,el,l),m(ie,e,l),s(e,ll,l),s(e,pe,l),tl=!0},p:Rl,i(e){tl||(w(f.$$.fragment,e),w(k.$$.fragment,e),w(R.$$.fragment,e),w(A.$$.fragment,e),w(v.$$.fragment,e),w(x.$$.fragment,e),w(X.$$.fragment,e),w(W.$$.fragment,e),w(Q.$$.fragment,e),w(z.$$.fragment,e),w(H.$$.fragment,e),w(q.$$.fragment,e),w(D.$$.fragment,e),w(O.$$.fragment,e),w(ee.$$.fragment,e),w(te.$$.fragment,e),w(ae.$$.fragment,e),w(ie.$$.fragment,e),tl=!0)},o(e){c(f.$$.fragment,e),c(k.$$.fragment,e),c(R.$$.fragment,e),c(A.$$.fragment,e),c(v.$$.fragment,e),c(x.$$.fragment,e),c(X.$$.fragment,e),c(W.$$.fragment,e),c(Q.$$.fragment,e),c(z.$$.fragment,e),c(H.$$.fragment,e),c(q.$$.fragment,e),c(D.$$.fragment,e),c(O.$$.fragment,e),c(ee.$$.fragment,e),c(te.$$.fragment,e),c(ae.$$.fragment,e),c(ie.$$.fragment,e),tl=!1},d(e){e&&(t(ye),t(Me),t(me),t(we),t(C),t(ce),t(I),t(de),t(h),t(Ue),t(b),t(Je),t(g),t(Te),t(G),t(je),t(B),t(ue),t(Z),t(he),t(fe),t(Ce),t(Ie),t($),t(be),t(ge),t(_),t(Ge),t(Be),t(Ze),t(E),t(ke),t(Re),t(F),t(Ae),t(Y),t($e),t(S),t(ve),t(_e),t(V),t(xe),t(Xe),t(N),t(Ee),t(We),t(L),t(Fe),t(Ye),t(P),t(Se),t(U),t(Qe),t(Ve),t(K),t(ze),t(Ne),t(J),t(He),t(Le),t(le),t(qe),t(Pe),t(T),t(De),t(se),t(Ke),t(Oe),t(ne),t(el),t(ll),t(pe)),t(u),d(f,e),d(k,e),d(R,e),d(A,e),d(v,e),d(x,e),d(X,e),d(W,e),d(Q,e),d(z,e),d(H,e),d(q,e),d(D,e),d(O,e),d(ee,e),d(te,e),d(ae,e),d(ie,e)}}}const Wl='{"title":"Implementing GRPO in TRL","local":"implementing-grpo-in-trl","sections":[{"title":"Key Components","local":"key-components","sections":[{"title":"1. Dataset Format","local":"1-dataset-format","sections":[],"depth":3},{"title":"2. Reward Function","local":"2-reward-function","sections":[],"depth":3},{"title":"3. Training Configuration","local":"3-training-configuration","sections":[],"depth":3}],"depth":2},{"title":"Tips for Success","local":"tips-for-success","sections":[],"depth":2},{"title":"Reward Function Design","local":"reward-function-design","sections":[{"title":"1. Length-Based Rewards","local":"1-length-based-rewards","sections":[],"depth":3}],"depth":2},{"title":"2. Rule-Based Rewards for Verifiable Tasks","local":"2-rule-based-rewards-for-verifiable-tasks","sections":[],"depth":2},{"title":"3. Format-Based Rewards","local":"3-format-based-rewards","sections":[],"depth":2},{"title":"That’s it!","local":"thats-it","sections":[],"depth":2}],"depth":1}';function Fl(al){return Al(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class zl extends $l{constructor(u){super(),vl(this,u,Fl,El,kl,{})}}export{zl as component};