course/pr_1007/en/_app/immutable/nodes/32.deff7011.js

import{s as $t,f as at,o as Rt,n as At}from"../chunks/scheduler.37c15a92.js";import{S as kt,i as vt,g as i,s as a,r as o,A as _t,h as r,f as l,c as n,j as ye,u as m,x as p,k as M,y as xt,a as s,v as y,d as w,t as c,w as d}from"../chunks/index.7cb9c9b8.js";import{T as Xt}from"../chunks/Tip.d10b3fc9.js";import{C as oe}from"../chunks/CodeBlock.abae2786.js";import{H as h,E as Et}from"../chunks/getInferenceSnippets.b6a8c7d2.js";function Wt(we){let U,C='This chapter is aimed at TRL beginners. If you are already familiar with TRL, you might want to also check out the <a href="https://github.com/huggingface/open-r1/blob/main/src/open_r1/grpo.py" rel="nofollow">Open R1 implementation</a> of GRPO.';return{c(){U=i("p"),U.innerHTML=C},l(u){U=r(u,"P",{"data-svelte-h":!0}),p(U)!=="svelte-14oapf3"&&(U.innerHTML=C)},m(u,Me){s(u,U,Me)},p:At,d(u){u&&l(U)}}}function Ft(we){let U,C,u,Me,I,ce,b,nt="In this page, we’ll learn how to implement Group Relative Policy Optimization (GRPO) using the Transformer Reinforcement Learning (TRL) library. We’ll focus on practical implementation with minimal code.",de,g,it="We’ll explore the core concepts of GRPO as they are embodied in TRL’s GRPOTrainer, using snippets from the official TRL documentation to guide us.",Ue,f,Je,G,rt="First, let’s remind ourselves of some of the important concepts of GRPO algorithm:",Te,B,pt="<li>Group Formation: The model generates multiple completions for each prompt.</li> <li>Preference Learning: The model learns from a reward function that compares groups of completions.</li> <li>Training Configuration: The model uses a configuration to control the training process.</li>",je,Z,Mt="What do we need to do to implement GRPO?",ue,$,ot="<li>Define a dataset of prompts.</li> <li>Define a reward function that takes a list of completions and returns a list of rewards.</li> <li>Configure the training process with a GRPOConfig.</li> <li>Train the model using the GRPOTrainer.</li>",he,R,mt="Here’s a minimal example to get started with GRPO training:",fe,A,Ce,k,Ie,v,be,_,yt="Your dataset should contain prompts that the model will respond to. The GRPO trainer will generate multiple completions for each prompt and use the reward function to compare them.",ge,x,Ge,X,wt="The reward function is crucial - it determines how the model learns. Here are two practical examples:",Be,E,Ze,W,$e,F,ct="Key parameters to consider in <code>GRPOConfig</code>:",Re,Y,Ae,S,dt="The <code>num_generation</code> parameter is particularly important for GRPO as it defines the group size - how many different completions the model will generate for each prompt. This is a key differentiator from other RL methods:",ke,V,Ut="<li>Too small (e.g., 2-3): May not provide enough diversity for meaningful comparisons</li> <li>Recommended (4-16): Provides good balance between diversity and computational efficiency</li> <li>Larger values: May improve learning but significantly increases computational cost</li>",ve,Q,Jt="The group size should be chosen based on your computational resources and the complexity of your task. For simple tasks, smaller groups (4-8) may be sufficient, while more complex reasoning tasks might benefit from larger groups (8-16).",_e,z,xe,N,Tt="<li><strong>Memory Management</strong>: Adjust <code>per_device_train_batch_size</code> and <code>gradient_accumulation_steps</code> based on your GPU memory.</li> <li><strong>Speed</strong>: Enable <code>use_vllm=True</code> for faster generation if your model is supported.</li> <li><strong>Monitoring</strong>: Watch the logged metrics during training:<ul><li><code>reward</code>: Average reward across completions</li> <li><code>reward_std</code>: Standard deviation within reward groups</li> <li><code>kl</code>: KL divergence from reference model</li></ul></li>",Xe,H,Ee,L,jt="The DeepSeek R1 paper demonstrates several effective approaches to reward function design that you can adapt for your own GRPO implementation:",We,P,Fe,q,ut="One of the easiest rewards to implement is a length-based reward. You can reward longer completions:",Ye,D,Se,K,ht="This reward function penalizes completions that are too short or too long, encouraging the model to generate completions that are close to the ideal length of 20 tokens.",Ve,J,ft,Qe,O,ze,ee,Ct="For tasks with objectively correct answers (like mathematics or coding), you can implement rule-based reward functions:",Ne,te,He,T,It,Le,le,Pe,se,bt="You can also reward proper formatting, which was important in the DeepSeek R1 training:",qe,ae,De,j,gt,Ke,ne,Gt="These examples demonstrate how you can implement reward functions inspired by the DeepSeek R1 training process, focusing on correctness, formatting, and combined signals.",Oe,ie,et,re,Bt="In the next section, you will follow an exercise to implement GRPO in TRL.",tt,pe,lt,me,st;return I=new h({props:{title:"Implementing GRPO in TRL",local:"implementing-grpo-in-trl",headingTag:"h1"}}),f=new Xt({props:{$$slots:{default:[Wt]},$$scope:{ctx:we}}}),A=new oe({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTJDJTIwR1JQT0NvbmZpZyUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQSUwQSUyMyUyMDEuJTIwTG9hZCUyMHlvdXIlMjBkYXRhc2V0JTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ5b3VyX2RhdGFzZXQlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQSUwQSUyMyUyMDIuJTIwRGVmaW5lJTIwYSUyMHNpbXBsZSUyMHJld2FyZCUyMGZ1bmN0aW9uJTBBZGVmJTIwcmV3YXJkX2Z1bmMoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjJFeGFtcGxlJTNBJTIwUmV3YXJkJTIwbG9uZ2VyJTIwY29tcGxldGlvbnMlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmbG9hdChsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMEElMEElMjMlMjAzLiUyMENvbmZpZ3VyZSUyMHRyYWluaW5nJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm91dHB1dCUyMiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QzJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDIlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEMTAlMkMlMEEpJTBBJTBBJTIzJTIwNC4lMjBJbml0aWFsaXplJTIwYW5kJTIwdHJhaW4lMEF0cmFpbmVyJTIwJTNEJTIwR1JQT1RyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJ5b3VyX21vZGVsJTIyJTJDJTIwJTIwJTIzJTIwZS5nLiUyMCUyMlF3ZW4lMkZRd2VuMi0wLjVCLUluc3RydWN0JTIyJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRHJld2FyZF9mdW5jJTJDJTBBKSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer, GRPOConfig
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># 1. Load your dataset</span>
dataset = load_dataset(<span class="hljs-string">&quot;your_dataset&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)


<span class="hljs-comment"># 2. Define a simple reward function</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Example: Reward longer completions&quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]


<span class="hljs-comment"># 3. Configure training</span>
training_args = GRPOConfig(
    output_dir=<span class="hljs-string">&quot;output&quot;</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    per_device_train_batch_size=<span class="hljs-number">4</span>,
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    logging_steps=<span class="hljs-number">10</span>,
)

<span class="hljs-comment"># 4. Initialize and train</span>
trainer = GRPOTrainer(
    model=<span class="hljs-string">&quot;your_model&quot;</span>,  <span class="hljs-comment"># e.g. &quot;Qwen/Qwen2-0.5B-Instruct&quot;</span>
    args=training_args,
    train_dataset=dataset,
    reward_funcs=reward_func,
)
trainer.train()`,wrap:!1}}),k=new h({props:{title:"Key Components",local:"key-components",headingTag:"h2"}}),v=new h({props:{title:"1. Dataset Format",local:"1-dataset-format",headingTag:"h3"}}),x=new h({props:{title:"2. Reward Function",local:"2-reward-function",headingTag:"h3"}}),E=new oe({props:{code:"JTIzJTIwRXhhbXBsZSUyMDElM0ElMjBSZXdhcmQlMjBiYXNlZCUyMG9uJTIwY29tcGxldGlvbiUyMGxlbmd0aCUwQWRlZiUyMHJld2FyZF9sZW5ndGgoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmbG9hdChsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMEElMEElMjMlMjBFeGFtcGxlJTIwMiUzQSUyMFJld2FyZCUyMGJhc2VkJTIwb24lMjBtYXRjaGluZyUyMGElMjBwYXR0ZXJuJTBBaW1wb3J0JTIwcmUlMEElMEElMEFkZWYlMjByZXdhcmRfZm9ybWF0KGNvbXBsZXRpb25zJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwcGF0dGVybiUyMCUzRCUyMHIlMjIlNUUlM0N0aGluayUzRS4qJTNGJTNDJTJGdGhpbmslM0UlM0NhbnN3ZXIlM0UuKiUzRiUzQyUyRmFuc3dlciUzRSUyNCUyMiUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QjEuMCUyMGlmJTIwcmUubWF0Y2gocGF0dGVybiUyQyUyMGMpJTIwZWxzZSUyMDAuMCUyMGZvciUyMGMlMjBpbiUyMGNvbXBsZXRpb25zJTVE",highlighted:`<span class="hljs-comment"># Example 1: Reward based on completion length</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_length</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]


<span class="hljs-comment"># Example 2: Reward based on matching a pattern</span>
<span class="hljs-keyword">import</span> re


<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_format</span>(<span class="hljs-params">completions, **kwargs</span>):
    pattern = <span class="hljs-string">r&quot;^&lt;think&gt;.*?&lt;/think&gt;&lt;answer&gt;.*?&lt;/answer&gt;$&quot;</span>
    <span class="hljs-keyword">return</span> [<span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> re.<span class="hljs-keyword">match</span>(pattern, c) <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),W=new h({props:{title:"3. Training Configuration",local:"3-training-configuration",headingTag:"h3"}}),Y=new oe({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwJTIzJTIwRXNzZW50aWFsJTIwcGFyYW1ldGVycyUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJvdXRwdXQlMjIlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMyUyQyUwQSUyMCUyMCUyMCUyMG51bV9nZW5lcmF0aW9uJTNENCUyQyUyMCUyMCUyMyUyME51bWJlciUyMG9mJTIwY29tcGxldGlvbnMlMjB0byUyMGdlbmVyYXRlJTIwZm9yJTIwZWFjaCUyMHByb21wdCUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDQlMkMlMjAlMjAlMjMlMjBXZSUyMHdhbnQlMjB0byUyMGdldCUyMGFsbCUyMGdlbmVyYXRpb25zJTIwaW4lMjBvbmUlMjBkZXZpY2UlMjBiYXRjaCUwQSUyMCUyMCUyMCUyMCUyMyUyME9wdGlvbmFsJTIwYnV0JTIwdXNlZnVsJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNEMiUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QxZS01JTJDJTBBJTIwJTIwJTIwJTIwbG9nZ2luZ19zdGVwcyUzRDEwJTJDJTBBJTIwJTIwJTIwJTIwJTIzJTIwR1JQTyUyMHNwZWNpZmljJTIwKG9wdGlvbmFsKSUwQSUyMCUyMCUyMCUyMHVzZV92bGxtJTNEVHJ1ZSUyQyUyMCUyMCUyMyUyMFNwZWVkJTIwdXAlMjBnZW5lcmF0aW9uJTBBKQ==",highlighted:`training_args = GRPOConfig(
    <span class="hljs-comment"># Essential parameters</span>
    output_dir=<span class="hljs-string">&quot;output&quot;</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    num_generation=<span class="hljs-number">4</span>,  <span class="hljs-comment"># Number of completions to generate for each prompt</span>
    per_device_train_batch_size=<span class="hljs-number">4</span>,  <span class="hljs-comment"># We want to get all generations in one device batch</span>
    <span class="hljs-comment"># Optional but useful</span>
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    learning_rate=<span class="hljs-number">1e-5</span>,
    logging_steps=<span class="hljs-number">10</span>,
    <span class="hljs-comment"># GRPO specific (optional)</span>
    use_vllm=<span class="hljs-literal">True</span>,  <span class="hljs-comment"># Speed up generation</span>
)`,wrap:!1}}),z=new h({props:{title:"Tips for Success",local:"tips-for-success",headingTag:"h2"}}),H=new h({props:{title:"Reward Function Design",local:"reward-function-design",headingTag:"h2"}}),P=new h({props:{title:"1. Length-Based Rewards",local:"1-length-based-rewards",headingTag:"h3"}}),D=new oe({props:{code:"ZGVmJTIwcmV3YXJkX2xlbihjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMGlkZWFsX2xlbmd0aCUyMCUzRCUyMDIwJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTVCLWFicyhpZGVhbF9sZW5ndGglMjAtJTIwbGVuKGNvbXBsZXRpb24pKSUyMGZvciUyMGNvbXBsZXRpb24lMjBpbiUyMGNvbXBsZXRpb25zJTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_len</span>(<span class="hljs-params">completions, **kwargs</span>):
    ideal_length = <span class="hljs-number">20</span>
    <span class="hljs-keyword">return</span> [-<span class="hljs-built_in">abs</span>(ideal_length - <span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),O=new h({props:{title:"2. Rule-Based Rewards for Verifiable Tasks",local:"2-rule-based-rewards-for-verifiable-tasks",headingTag:"h2"}}),te=new oe({props:{code:"ZGVmJTIwcHJvYmxlbV9yZXdhcmQoY29tcGxldGlvbnMlMkMlMjBhbnN3ZXJzJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyUmV3YXJkJTIwZnVuY3Rpb24lMjBmb3IlMjBtYXRoJTIwcHJvYmxlbXMlMjB3aXRoJTIwdmVyaWZpYWJsZSUyMGFuc3dlcnMlMEElMjAlMjAlMjAlMjBjb21wbGV0aW9ucyUzQSUyMGxpc3QlMjBvZiUyMGNvbXBsZXRpb25zJTIwdG8lMjBldmFsdWF0ZSUwQSUyMCUyMCUyMCUyMGFuc3dlcnMlM0ElMjBsaXN0JTIwb2YlMjBhbnN3ZXJzJTIwdG8lMjB0aGUlMjBwcm9ibGVtcyUyMGZyb20lMjB0aGUlMjBkYXRhc2V0JTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTBBJTIwJTIwJTIwJTIwcmV3YXJkcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGNvbXBsZXRpb24lMkMlMjBjb3JyZWN0X2Fuc3dlciUyMGluJTIwemlwKGNvbXBsZXRpb25zJTJDJTIwYW5zd2VycyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBFeHRyYWN0JTIwdGhlJTIwYW5zd2VyJTIwZnJvbSUyMHRoZSUyMGNvbXBsZXRpb24lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBUaGlzJTIwaXMlMjBhJTIwc2ltcGxpZmllZCUyMGV4YW1wbGUlMjAtJTIweW91J2QlMjBuZWVkJTIwcHJvcGVyJTIwcGFyc2luZyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFuc3dlciUyMCUzRCUyMGV4dHJhY3RfZmluYWxfYW5zd2VyKGNvbXBsZXRpb24pJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwQmluYXJ5JTIwcmV3YXJkJTNBJTIwMSUyMGZvciUyMGNvcnJlY3QlMkMlMjAwJTIwZm9yJTIwaW5jb3JyZWN0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV3YXJkJTIwJTNEJTIwMS4wJTIwaWYlMjBhbnN3ZXIlMjAlM0QlM0QlMjBjb3JyZWN0X2Fuc3dlciUyMGVsc2UlMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZChyZXdhcmQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXhjZXB0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwSWYlMjB3ZSUyMGNhbid0JTIwcGFyc2UlMjBhbiUyMGFuc3dlciUyQyUyMGdpdmUlMjBhJTIwbG93JTIwcmV3YXJkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV3YXJkcy5hcHBlbmQoMC4wKSUwQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJld2FyZHM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">problem_reward</span>(<span class="hljs-params">completions, answers, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Reward function for math problems with verifiable answers
    completions: list of completions to evaluate
    answers: list of answers to the problems from the dataset
    &quot;&quot;&quot;</span>

    rewards = []
    <span class="hljs-keyword">for</span> completion, correct_answer <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(completions, answers):
        <span class="hljs-comment"># Extract the answer from the completion</span>
        <span class="hljs-keyword">try</span>:
            <span class="hljs-comment"># This is a simplified example - you&#x27;d need proper parsing</span>
            answer = extract_final_answer(completion)
            <span class="hljs-comment"># Binary reward: 1 for correct, 0 for incorrect</span>
            reward = <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> answer == correct_answer <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span>
            rewards.append(reward)
        <span class="hljs-keyword">except</span>:
            <span class="hljs-comment"># If we can&#x27;t parse an answer, give a low reward</span>
            rewards.append(<span class="hljs-number">0.0</span>)

    <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),le=new h({props:{title:"3. Format-Based Rewards",local:"3-format-based-rewards",headingTag:"h2"}}),ae=new oe({props:{code:"ZGVmJTIwZm9ybWF0X3Jld2FyZChjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMlJld2FyZCUyMGNvbXBsZXRpb25zJTIwdGhhdCUyMGZvbGxvdyUyMHRoZSUyMGRlc2lyZWQlMjBmb3JtYXQlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjAlMjMlMjBFeGFtcGxlJTNBJTIwQ2hlY2slMjBpZiUyMHRoZSUyMGNvbXBsZXRpb24lMjBmb2xsb3dzJTIwYSUyMHRoaW5rLXRoZW4tYW5zd2VyJTIwZm9ybWF0JTBBJTIwJTIwJTIwJTIwcGF0dGVybiUyMCUzRCUyMHIlMjIlM0N0aGluayUzRSguKiUzRiklM0MlMkZ0aGluayUzRSU1Q3MqJTNDYW5zd2VyJTNFKC4qJTNGKSUzQyUyRmFuc3dlciUzRSUyMiUwQSUwQSUyMCUyMCUyMCUyMHJld2FyZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBjb21wbGV0aW9uJTIwaW4lMjBjb21wbGV0aW9ucyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hdGNoJTIwJTNEJTIwcmUuc2VhcmNoKHBhdHRlcm4lMkMlMjBjb21wbGV0aW9uJTJDJTIwcmUuRE9UQUxMKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwbWF0Y2glM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBDaGVjayUyMGlmJTIwdGhlcmUncyUyMHN1YnN0YW50aWFsJTIwY29udGVudCUyMGluJTIwYm90aCUyMHNlY3Rpb25zJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdGhpbmtfY29udGVudCUyMCUzRCUyMG1hdGNoLmdyb3VwKDEpLnN0cmlwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXJfY29udGVudCUyMCUzRCUyMG1hdGNoLmdyb3VwKDIpLnN0cmlwKCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGxlbih0aGlua19jb250ZW50KSUyMCUzRSUyMDIwJTIwYW5kJTIwbGVuKGFuc3dlcl9jb250ZW50KSUyMCUzRSUyMDAlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZCgxLjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZHMuYXBwZW5kKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDAuNSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMjAlMjAlMjMlMjBQYXJ0aWFsJTIwcmV3YXJkJTIwZm9yJTIwY29ycmVjdCUyMGZvcm1hdCUyMGJ1dCUyMGxpbWl0ZWQlMjBjb250ZW50JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZHMuYXBwZW5kKDAuMCklMjAlMjAlMjMlMjBObyUyMHJld2FyZCUyMGZvciUyMGluY29ycmVjdCUyMGZvcm1hdCUwQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJld2FyZHM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">format_reward</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Reward completions that follow the desired format&quot;&quot;&quot;</span>
    <span class="hljs-comment"># Example: Check if the completion follows a think-then-answer format</span>
    pattern = <span class="hljs-string">r&quot;&lt;think&gt;(.*?)&lt;/think&gt;\\s*&lt;answer&gt;(.*?)&lt;/answer&gt;&quot;</span>

    rewards = []
    <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions:
        <span class="hljs-keyword">match</span> = re.search(pattern, completion, re.DOTALL)
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span>:
            <span class="hljs-comment"># Check if there&#x27;s substantial content in both sections</span>
            think_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">1</span>).strip()
            answer_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">2</span>).strip()

            <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(think_content) &gt; <span class="hljs-number">20</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(answer_content) &gt; <span class="hljs-number">0</span>:
                rewards.append(<span class="hljs-number">1.0</span>)
            <span class="hljs-keyword">else</span>:
                rewards.append(
                    <span class="hljs-number">0.5</span>
                )  <span class="hljs-comment"># Partial reward for correct format but limited content</span>
        <span class="hljs-keyword">else</span>:
            rewards.append(<span class="hljs-number">0.0</span>)  <span class="hljs-comment"># No reward for incorrect format</span>

    <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),ie=new h({props:{title:"That’s it!",local:"thats-it",headingTag:"h2"}}),pe=new Et({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter12/4.mdx"}}),{c(){U=i("meta"),C=a(),u=i("p"),Me=a(),o(I.$$.fragment),ce=a(),b=i("p"),b.textContent=nt,de=a(),g=i("p"),g.textContent=it,Ue=a(),o(f.$$.fragment),Je=a(),G=i("p"),G.textContent=rt,Te=a(),B=i("ul"),B.innerHTML=pt,je=a(),Z=i("p"),Z.textContent=Mt,ue=a(),$=i("ul"),$.innerHTML=ot,he=a(),R=i("p"),R.textContent=mt,fe=a(),o(A.$$.fragment),Ce=a(),o(k.$$.fragment),Ie=a(),o(v.$$.fragment),be=a(),_=i("p"),_.textContent=yt,ge=a(),o(x.$$.fragment),Ge=a(),X=i("p"),X.textContent=wt,Be=a(),o(E.$$.fragment),Ze=a(),o(W.$$.fragment),$e=a(),F=i("p"),F.innerHTML=ct,Re=a(),o(Y.$$.fragment),Ae=a(),S=i("p"),S.innerHTML=dt,ke=a(),V=i("ul"),V.innerHTML=Ut,ve=a(),Q=i("p"),Q.textContent=Jt,_e=a(),o(z.$$.fragment),xe=a(),N=i("ol"),N.innerHTML=Tt,Xe=a(),o(H.$$.fragment),Ee=a(),L=i("p"),L.textContent=jt,We=a(),o(P.$$.fragment),Fe=a(),q=i("p"),q.textContent=ut,Ye=a(),o(D.$$.fragment),Se=a(),K=i("p"),K.textContent=ht,Ve=a(),J=i("iframe"),Qe=a(),o(O.$$.fragment),ze=a(),ee=i("p"),ee.textContent=Ct,Ne=a(),o(te.$$.fragment),He=a(),T=i("iframe"),Le=a(),o(le.$$.fragment),Pe=a(),se=i("p"),se.textContent=bt,qe=a(),o(ae.$$.fragment),De=a(),j=i("iframe"),Ke=a(),ne=i("p"),ne.textContent=Gt,Oe=a(),o(ie.$$.fragment),et=a(),re=i("p"),re.textContent=Bt,tt=a(),o(pe.$$.fragment),lt=a(),me=i("p"),this.h()},l(e){const t=_t("svelte-u9bgzb",document.head);U=r(t,"META",{name:!0,content:!0}),t.forEach(l),C=n(e),u=r(e,"P",{}),ye(u).forEach(l),Me=n(e),m(I.$$.fragment,e),ce=n(e),b=r(e,"P",{"data-svelte-h":!0}),p(b)!=="svelte-v9iq5e"&&(b.textContent=nt),de=n(e),g=r(e,"P",{"data-svelte-h":!0}),p(g)!=="svelte-iqdzey"&&(g.textContent=it),Ue=n(e),m(f.$$.fragment,e),Je=n(e),G=r(e,"P",{"data-svelte-h":!0}),p(G)!=="svelte-14hk95q"&&(G.textContent=rt),Te=n(e),B=r(e,"UL",{"data-svelte-h":!0}),p(B)!=="svelte-1vd6wz6"&&(B.innerHTML=pt),je=n(e),Z=r(e,"P",{"data-svelte-h":!0}),p(Z)!=="svelte-pzoe2y"&&(Z.textContent=Mt),ue=n(e),$=r(e,"UL",{"data-svelte-h":!0}),p($)!=="svelte-1hif2as"&&($.innerHTML=ot),he=n(e),R=r(e,"P",{"data-svelte-h":!0}),p(R)!=="svelte-ac9u4w"&&(R.textContent=mt),fe=n(e),m(A.$$.fragment,e),Ce=n(e),m(k.$$.fragment,e),Ie=n(e),m(v.$$.fragment,e),be=n(e),_=r(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-1pnmwnq"&&(_.textContent=yt),ge=n(e),m(x.$$.fragment,e),Ge=n(e),X=r(e,"P",{"data-svelte-h":!0}),p(X)!=="svelte-ths948"&&(X.textContent=wt),Be=n(e),m(E.$$.fragment,e),Ze=n(e),m(W.$$.fragment,e),$e=n(e),F=r(e,"P",{"data-svelte-h":!0}),p(F)!=="svelte-1uukcnx"&&(F.innerHTML=ct),Re=n(e),m(Y.$$.fragment,e),Ae=n(e),S=r(e,"P",{"data-svelte-h":!0}),p(S)!=="svelte-1tdjaxe"&&(S.innerHTML=dt),ke=n(e),V=r(e,"UL",{"data-svelte-h":!0}),p(V)!=="svelte-1b1t69f"&&(V.innerHTML=Ut),ve=n(e),Q=r(e,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-7howml"&&(Q.textContent=Jt),_e=n(e),m(z.$$.fragment,e),xe=n(e),N=r(e,"OL",{"data-svelte-h":!0}),p(N)!=="svelte-4ukx5l"&&(N.innerHTML=Tt),Xe=n(e),m(H.$$.fragment,e),Ee=n(e),L=r(e,"P",{"data-svelte-h":!0}),p(L)!=="svelte-5lvuy8"&&(L.textContent=jt),We=n(e),m(P.$$.fragment,e),Fe=n(e),q=r(e,"P",{"data-svelte-h":!0}),p(q)!=="svelte-jhf03w"&&(q.textContent=ut),Ye=n(e),m(D.$$.fragment,e),Se=n(e),K=r(e,"P",{"data-svelte-h":!0}),p(K)!=="svelte-jlmna3"&&(K.textContent=ht),Ve=n(e),J=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),ye(J).forEach(l),Qe=n(e),m(O.$$.fragment,e),ze=n(e),ee=r(e,"P",{"data-svelte-h":!0}),p(ee)!=="svelte-mrpax"&&(ee.textContent=Ct),Ne=n(e),m(te.$$.fragment,e),He=n(e),T=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),ye(T).forEach(l),Le=n(e),m(le.$$.fragment,e),Pe=n(e),se=r(e,"P",{"data-svelte-h":!0}),p(se)!=="svelte-18k4wdv"&&(se.textContent=bt),qe=n(e),m(ae.$$.fragment,e),De=n(e),j=r(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),ye(j).forEach(l),Ke=n(e),ne=r(e,"P",{"data-svelte-h":!0}),p(ne)!=="svelte-fzl1fe"&&(ne.textContent=Gt),Oe=n(e),m(ie.$$.fragment,e),et=n(e),re=r(e,"P",{"data-svelte-h":!0}),p(re)!=="svelte-chd5s6"&&(re.textContent=Bt),tt=n(e),m(pe.$$.fragment,e),lt=n(e),me=r(e,"P",{}),ye(me).forEach(l),this.h()},h(){M(U,"name","hf:doc:metadata"),M(U,"content",Yt),at(J.src,ft="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_length.py&embed=true&show-chrome=false")||M(J,"src",ft),M(J,"title","Marimo Notebook"),M(J,"width","100%"),M(J,"height","800px"),M(J,"frameborder","0"),M(J,"allow","clipboard-write"),at(T.src,It="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_math.py&embed=true&show-chrome=false")||M(T,"src",It),M(T,"title","Marimo Notebook"),M(T,"width","100%"),M(T,"height","800px"),M(T,"frameborder","0"),M(T,"allow","clipboard-write"),at(j.src,gt="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_format.py&embed=true&show-chrome=false")||M(j,"src",gt),M(j,"title","Marimo Notebook"),M(j,"width","100%"),M(j,"height","800px"),M(j,"frameborder","0"),M(j,"allow","clipboard-write")},m(e,t){xt(document.head,U),s(e,C,t),s(e,u,t),s(e,Me,t),y(I,e,t),s(e,ce,t),s(e,b,t),s(e,de,t),s(e,g,t),s(e,Ue,t),y(f,e,t),s(e,Je,t),s(e,G,t),s(e,Te,t),s(e,B,t),s(e,je,t),s(e,Z,t),s(e,ue,t),s(e,$,t),s(e,he,t),s(e,R,t),s(e,fe,t),y(A,e,t),s(e,Ce,t),y(k,e,t),s(e,Ie,t),y(v,e,t),s(e,be,t),s(e,_,t),s(e,ge,t),y(x,e,t),s(e,Ge,t),s(e,X,t),s(e,Be,t),y(E,e,t),s(e,Ze,t),y(W,e,t),s(e,$e,t),s(e,F,t),s(e,Re,t),y(Y,e,t),s(e,Ae,t),s(e,S,t),s(e,ke,t),s(e,V,t),s(e,ve,t),s(e,Q,t),s(e,_e,t),y(z,e,t),s(e,xe,t),s(e,N,t),s(e,Xe,t),y(H,e,t),s(e,Ee,t),s(e,L,t),s(e,We,t),y(P,e,t),s(e,Fe,t),s(e,q,t),s(e,Ye,t),y(D,e,t),s(e,Se,t),s(e,K,t),s(e,Ve,t),s(e,J,t),s(e,Qe,t),y(O,e,t),s(e,ze,t),s(e,ee,t),s(e,Ne,t),y(te,e,t),s(e,He,t),s(e,T,t),s(e,Le,t),y(le,e,t),s(e,Pe,t),s(e,se,t),s(e,qe,t),y(ae,e,t),s(e,De,t),s(e,j,t),s(e,Ke,t),s(e,ne,t),s(e,Oe,t),y(ie,e,t),s(e,et,t),s(e,re,t),s(e,tt,t),y(pe,e,t),s(e,lt,t),s(e,me,t),st=!0},p(e,[t]){const Zt={};t&2&&(Zt.$$scope={dirty:t,ctx:e}),f.$set(Zt)},i(e){st||(w(I.$$.fragment,e),w(f.$$.fragment,e),w(A.$$.fragment,e),w(k.$$.fragment,e),w(v.$$.fragment,e),w(x.$$.fragment,e),w(E.$$.fragment,e),w(W.$$.fragment,e),w(Y.$$.fragment,e),w(z.$$.fragment,e),w(H.$$.fragment,e),w(P.$$.fragment,e),w(D.$$.fragment,e),w(O.$$.fragment,e),w(te.$$.fragment,e),w(le.$$.fragment,e),w(ae.$$.fragment,e),w(ie.$$.fragment,e),w(pe.$$.fragment,e),st=!0)},o(e){c(I.$$.fragment,e),c(f.$$.fragment,e),c(A.$$.fragment,e),c(k.$$.fragment,e),c(v.$$.fragment,e),c(x.$$.fragment,e),c(E.$$.fragment,e),c(W.$$.fragment,e),c(Y.$$.fragment,e),c(z.$$.fragment,e),c(H.$$.fragment,e),c(P.$$.fragment,e),c(D.$$.fragment,e),c(O.$$.fragment,e),c(te.$$.fragment,e),c(le.$$.fragment,e),c(ae.$$.fragment,e),c(ie.$$.fragment,e),c(pe.$$.fragment,e),st=!1},d(e){e&&(l(C),l(u),l(Me),l(ce),l(b),l(de),l(g),l(Ue),l(Je),l(G),l(Te),l(B),l(je),l(Z),l(ue),l($),l(he),l(R),l(fe),l(Ce),l(Ie),l(be),l(_),l(ge),l(Ge),l(X),l(Be),l(Ze),l($e),l(F),l(Re),l(Ae),l(S),l(ke),l(V),l(ve),l(Q),l(_e),l(xe),l(N),l(Xe),l(Ee),l(L),l(We),l(Fe),l(q),l(Ye),l(Se),l(K),l(Ve),l(J),l(Qe),l(ze),l(ee),l(Ne),l(He),l(T),l(Le),l(Pe),l(se),l(qe),l(De),l(j),l(Ke),l(ne),l(Oe),l(et),l(re),l(tt),l(lt),l(me)),l(U),d(I,e),d(f,e),d(A,e),d(k,e),d(v,e),d(x,e),d(E,e),d(W,e),d(Y,e),d(z,e),d(H,e),d(P,e),d(D,e),d(O,e),d(te,e),d(le,e),d(ae,e),d(ie,e),d(pe,e)}}}const Yt='{"title":"Implementing GRPO in TRL","local":"implementing-grpo-in-trl","sections":[{"title":"Key Components","local":"key-components","sections":[{"title":"1. Dataset Format","local":"1-dataset-format","sections":[],"depth":3},{"title":"2. Reward Function","local":"2-reward-function","sections":[],"depth":3},{"title":"3. Training Configuration","local":"3-training-configuration","sections":[],"depth":3}],"depth":2},{"title":"Tips for Success","local":"tips-for-success","sections":[],"depth":2},{"title":"Reward Function Design","local":"reward-function-design","sections":[{"title":"1. Length-Based Rewards","local":"1-length-based-rewards","sections":[],"depth":3}],"depth":2},{"title":"2. Rule-Based Rewards for Verifiable Tasks","local":"2-rule-based-rewards-for-verifiable-tasks","sections":[],"depth":2},{"title":"3. Format-Based Rewards","local":"3-format-based-rewards","sections":[],"depth":2},{"title":"That’s it!","local":"thats-it","sections":[],"depth":2}],"depth":1}';function St(we){return Rt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Lt extends kt{constructor(U){super(),vt(this,U,St,Ft,$t,{})}}export{Lt as component};