course/pr_1069/en/_app/immutable/nodes/34.f41cea0d.js

import{s as Jt,o as mt,n as Vs}from"../chunks/scheduler.37c15a92.js";import{S as ut,i as Tt,g as M,s as n,r,A as Ut,h as p,f as t,c as a,j as jt,u as c,x as o,k as wt,y as ht,a as e,v as y,d as j,t as w,w as J}from"../chunks/index.7cb9c9b8.js";import{T as ks}from"../chunks/Tip.d10b3fc9.js";import{C as T}from"../chunks/CodeBlock.abae2786.js";import{C as dt}from"../chunks/CourseFloatingBanner.df82c153.js";import{H as h,E as bt}from"../chunks/getInferenceSnippets.f9350a3f.js";function It(d){let i,u="This exercise can be run on a free Google Colab T4 GPU. For the best experience, follow along with the notebook linked above and try it out yourself.";return{c(){i=M("p"),i.textContent=u},l(m){i=p(m,"P",{"data-svelte-h":!0}),o(i)!=="svelte-1tto42k"&&(i.textContent=u)},m(m,U){e(m,i,U)},p:Vs,d(m){m&&t(i)}}}function ft(d){let i,u='We won’t cover the details of LoRA in this chapter, but you can learn more in <a href="/course/chapter11/3">Chapter 11</a>.';return{c(){i=M("p"),i.innerHTML=u},l(m){i=p(m,"P",{"data-svelte-h":!0}),o(i)!=="svelte-1qfqkww"&&(i.innerHTML=u)},m(m,U){e(m,i,U)},p:Vs,d(m){m&&t(i)}}}function Ct(d){let i,u="Training may take some time. You might not see rewards increase immediately - it can take 150-200 steps before you start seeing improvements. Be patient!";return{c(){i=M("p"),i.textContent=u},l(m){i=p(m,"P",{"data-svelte-h":!0}),o(i)!=="svelte-1q4mcgt"&&(i.textContent=u)},m(m,U){e(m,i,U)},p:Vs,d(m){m&&t(i)}}}function gt(d){let i,u,m,U,C,Bl,g,Al,B,$s='In this exercise, you’ll fine-tune a model with GRPO (Group Relative Policy Optimization) using Unsloth, to improve a model’s reasoning capabilities. We covered GRPO in <a href="/course/chapter3/3">Chapter 3</a>.',Gl,A,Qs="Unsloth is a library that accelerates LLM fine-tuning, making it possible to train models faster and with less computational resources. Unsloth is plugs into TRL, so we’ll build on what we learned in the previous sections, and adapt it for Unsloth specifics.",_l,b,Zl,G,kl,_,Es="First, let’s install the necessary libraries. We’ll need Unsloth for the accelerated fine-tuning and vLLM for fast inference.",Vl,Z,$l,k,Ql,V,Rs="Unsloth provides a class (<code>FastLanguageModel</code>) that integrates transformers with Unsloth optimizations. Let’s import it:",El,$,Rl,Q,Ns="Now, let’s load Google’s Gemma 3 1B Instruct model and configure it for fine-tuning:",Nl,E,vl,R,vs="This code loads the model in 4-bit quantization to save memory and applies LoRA (Low-Rank Adaptation) for efficient fine-tuning. The <code>target_modules</code> parameter specifies which layers of the model to fine-tune, and <code>use_gradient_checkpointing</code> enables training with longer contexts.",xl,I,Xl,N,Wl,v,xs="For this exercise, we’ll use the GSM8K dataset, which contains grade school math problems. We’ll format the data to encourage the model to show its reasoning before providing an answer.",Yl,x,Xs="First, we will define the format of the prompts and answers:",ql,X,zl,W,Ws="Now, let’s prepare the dataset:",Fl,Y,Hl,q,Ys="The dataset is prepared by extracting the answer from the dataset and formatting it as a string.",Sl,z,Ll,F,qs='As we discussed in <a href="/course/chapter13/4">an earlier page</a>, GRPO can use reward functions to guide the model’s learning based on verifiable criteria like length and formatting.',Dl,H,zs="In this exercise, we’ll define several reward functions that encourage different aspects of good reasoning. For example, we’ll reward the model for providing an integer answer, and for following the strict format.",Pl,S,Kl,L,Fs="These reward functions serve different purposes:",Ol,D,Hs="<thead><tr><th>Reward Function</th> <th>Purpose</th></tr></thead> <tbody><tr><td><code>correctness_reward_func</code></td> <td>Rewards the model when its answer matches the correct answer</td></tr> <tr><td><code>int_reward_func</code></td> <td>Rewards the model for providing a numeric answer</td></tr> <tr><td><code>strict_format_reward_func</code> and <code>soft_format_reward_func</code></td> <td>Reward the model for following the specified format</td></tr> <tr><td><code>xmlcount_reward_func</code></td> <td>Rewards proper XML tag usage and penalizes extra content after the closing tags</td></tr></tbody>",ls,P,ss,K,Ss='Now we’ll set up the GRPO trainer with our model, tokenizer, and reward functions. This part follows the same approach as the <a href="/course/chapter12/5">previous exercise</a>.',ts,O,es,ll,Ls="The <code>GRPOConfig</code> sets various hyperparameters for training:",ns,sl,Ds="<li><code>use_vllm</code>: Enables fast inference with vLLM</li> <li><code>learning_rate</code>: Controls how quickly the model learns</li> <li><code>num_generations</code>: Number of completions to generate for each prompt</li> <li><code>max_steps</code>: Total number of training steps to perform</li>",as,tl,Ps="Now let’s start the training:",Ms,el,ps,f,is,nl,os,al,Ks="After training, let’s test our model to see how it performs. First, we’ll save the LoRA weights:",rs,Ml,cs,pl,Os="Now, let’s test the model with a new question:",ys,il,js,ol,lt="You should see that the model now follows the specified format, showing its reasoning before providing an answer.",ws,rl,Js,cl,st="Unsloth provides several options for saving your fine-tuned model, but we’ll focus on the most common.",ms,yl,us,jl,Ts,wl,tt="We’ll push the model to the Hugging Face Hub using the <code>push_to_hub_merged</code> method. This method allows us to push the model in multiple quantization formats.",Us,Jl,hs,ml,et="Unsloth also supports saving to GGUF format for use with llama.cpp:",ds,ul,bs,Tl,nt="The GGUF files can be used with llama.cpp or UI-based systems like Jan or Open WebUI.",Is,Ul,fs,hl,at="In this exercise, you’ve learned how to:",Cs,dl,Mt="<li>Set up Unsloth for accelerated fine-tuning</li> <li>Prepare data for GRPO training</li> <li>Define custom reward functions to guide the model’s learning</li> <li>Train a model using GRPO</li> <li>Test the fine-tuned model</li> <li>Save the model in various formats</li>",gs,bl,pt="GRPO is a powerful technique for aligning language models with specific behaviors, and Unsloth makes it accessible even on limited hardware. By combining multiple reward functions, you can guide the model to follow a specific format while also improving its reasoning capabilities.",Bs,Il,it="For more information and resources, check out:",As,fl,ot='<li><a href="https://docs.unsloth.ai/" rel="nofollow">Unsloth Documentation</a></li> <li><a href="https://discord.gg/unsloth" rel="nofollow">Unsloth Discord</a></li> <li><a href="https://github.com/unslothai/unsloth" rel="nofollow">Unsloth GitHub</a></li>',Gs,Cl,_s,gl,Zs;return C=new dt({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/HuggingFace%20Course-Gemma3_(1B)-GRPO.ipynb"}]}}),g=new h({props:{title:"Practical Exercise: GRPO with Unsloth",local:"practical-exercise-grpo-with-unsloth",headingTag:"h1"}}),b=new ks({props:{$$slots:{default:[It]},$$scope:{ctx:d}}}),G=new h({props:{title:"Install dependencies",local:"install-dependencies",headingTag:"h2"}}),Z=new T({props:{code:"cGlwJTIwaW5zdGFsbCUyMHVuc2xvdGglMjB2bGxtJTBBcGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMHBpbGxvdw==",highlighted:`pip install unsloth vllm
pip install --upgrade pillow`,wrap:!1}}),k=new h({props:{title:"Setting up Unsloth",local:"setting-up-unsloth",headingTag:"h2"}}),$=new T({props:{code:"ZnJvbSUyMHVuc2xvdGglMjBpbXBvcnQlMjBGYXN0TGFuZ3VhZ2VNb2RlbA==",highlighted:'<span class="hljs-keyword">from</span> unsloth <span class="hljs-keyword">import</span> FastLanguageModel',wrap:!1}}),E=new T({props:{code:"ZnJvbSUyMHVuc2xvdGglMjBpbXBvcnQlMjBGYXN0TGFuZ3VhZ2VNb2RlbCUwQWltcG9ydCUyMHRvcmNoJTBBJTBBbWF4X3NlcV9sZW5ndGglMjAlM0QlMjAxMDI0JTIwJTIwJTIzJTIwQ2FuJTIwaW5jcmVhc2UlMjBmb3IlMjBsb25nZXIlMjByZWFzb25pbmclMjB0cmFjZXMlMEFsb3JhX3JhbmslMjAlM0QlMjAzMiUyMCUyMCUyMyUyMExhcmdlciUyMHJhbmslMjAlM0QlMjBzbWFydGVyJTJDJTIwYnV0JTIwc2xvd2VyJTBBJTBBbW9kZWwlMkMlMjB0b2tlbml6ZXIlMjAlM0QlMjBGYXN0TGFuZ3VhZ2VNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwbW9kZWxfbmFtZSUzRCUyMmdvb2dsZSUyRmdlbW1hLTMtMWItaXQlMjIlMkMlMEElMjAlMjAlMjAlMjBtYXhfc2VxX2xlbmd0aCUzRG1heF9zZXFfbGVuZ3RoJTJDJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUyMCUyMCUyMyUyMEZhbHNlJTIwZm9yJTIwTG9SQSUyMDE2Yml0JTBBJTIwJTIwJTIwJTIwZmFzdF9pbmZlcmVuY2UlM0RUcnVlJTJDJTIwJTIwJTIzJTIwRW5hYmxlJTIwdkxMTSUyMGZhc3QlMjBpbmZlcmVuY2UlMEElMjAlMjAlMjAlMjBtYXhfbG9yYV9yYW5rJTNEbG9yYV9yYW5rJTJDJTBBJTIwJTIwJTIwJTIwZ3B1X21lbW9yeV91dGlsaXphdGlvbiUzRDAuNiUyQyUyMCUyMCUyMyUyMFJlZHVjZSUyMGlmJTIwb3V0JTIwb2YlMjBtZW1vcnklMEEpJTBBJTBBbW9kZWwlMjAlM0QlMjBGYXN0TGFuZ3VhZ2VNb2RlbC5nZXRfcGVmdF9tb2RlbCglMEElMjAlMjAlMjAlMjBtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMHIlM0Rsb3JhX3JhbmslMkMlMjAlMjAlMjMlMjBDaG9vc2UlMjBhbnklMjBudW1iZXIlMjAlM0UlMjAwJTIwISUyMFN1Z2dlc3RlZCUyMDglMkMlMjAxNiUyQyUyMDMyJTJDJTIwNjQlMkMlMjAxMjglMEElMjAlMjAlMjAlMjB0YXJnZXRfbW9kdWxlcyUzRCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnFfcHJvaiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmtfcHJvaiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnZfcHJvaiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm9fcHJvaiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmdhdGVfcHJvaiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnVwX3Byb2olMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJkb3duX3Byb2olMjIlMkMlMEElMjAlMjAlMjAlMjAlNUQlMkMlMjAlMjAlMjMlMjBSZW1vdmUlMjBRS1ZPJTIwaWYlMjBvdXQlMjBvZiUyMG1lbW9yeSUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0Rsb3JhX3JhbmslMkMlMEElMjAlMjAlMjAlMjB1c2VfZ3JhZGllbnRfY2hlY2twb2ludGluZyUzRCUyMnVuc2xvdGglMjIlMkMlMjAlMjAlMjMlMjBFbmFibGUlMjBsb25nJTIwY29udGV4dCUyMGZpbmV0dW5pbmclMEElMjAlMjAlMjAlMjByYW5kb21fc3RhdGUlM0QzNDA3JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> unsloth <span class="hljs-keyword">import</span> FastLanguageModel
<span class="hljs-keyword">import</span> torch

max_seq_length = <span class="hljs-number">1024</span>  <span class="hljs-comment"># Can increase for longer reasoning traces</span>
lora_rank = <span class="hljs-number">32</span>  <span class="hljs-comment"># Larger rank = smarter, but slower</span>

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name=<span class="hljs-string">&quot;google/gemma-3-1b-it&quot;</span>,
    max_seq_length=max_seq_length,
    load_in_4bit=<span class="hljs-literal">True</span>,  <span class="hljs-comment"># False for LoRA 16bit</span>
    fast_inference=<span class="hljs-literal">True</span>,  <span class="hljs-comment"># Enable vLLM fast inference</span>
    max_lora_rank=lora_rank,
    gpu_memory_utilization=<span class="hljs-number">0.6</span>,  <span class="hljs-comment"># Reduce if out of memory</span>
)

model = FastLanguageModel.get_peft_model(
    model,
    r=lora_rank,  <span class="hljs-comment"># Choose any number &gt; 0 ! Suggested 8, 16, 32, 64, 128</span>
    target_modules=[
        <span class="hljs-string">&quot;q_proj&quot;</span>,
        <span class="hljs-string">&quot;k_proj&quot;</span>,
        <span class="hljs-string">&quot;v_proj&quot;</span>,
        <span class="hljs-string">&quot;o_proj&quot;</span>,
        <span class="hljs-string">&quot;gate_proj&quot;</span>,
        <span class="hljs-string">&quot;up_proj&quot;</span>,
        <span class="hljs-string">&quot;down_proj&quot;</span>,
    ],  <span class="hljs-comment"># Remove QKVO if out of memory</span>
    lora_alpha=lora_rank,
    use_gradient_checkpointing=<span class="hljs-string">&quot;unsloth&quot;</span>,  <span class="hljs-comment"># Enable long context finetuning</span>
    random_state=<span class="hljs-number">3407</span>,
)`,wrap:!1}}),I=new ks({props:{$$slots:{default:[ft]},$$scope:{ctx:d}}}),N=new h({props:{title:"Data Preparation",local:"data-preparation",headingTag:"h2"}}),X=new T({props:{code:"",highlighted:`<span class="hljs-comment"># Define the system prompt that instructs the model to use a specific format</span>
SYSTEM_PROMPT = <span class="hljs-string">&quot;&quot;&quot;
Respond in the following format:
&lt;reasoning&gt;
...
&lt;/reasoning&gt;
&lt;answer&gt;
...
&lt;/answer&gt;
&quot;&quot;&quot;</span>

XML_COT_FORMAT = <span class="hljs-string">&quot;&quot;&quot;\\
&lt;reasoning&gt;
{reasoning}
&lt;/reasoning&gt;
&lt;answer&gt;
{answer}
&lt;/answer&gt;
&quot;&quot;&quot;</span>`,wrap:!1}}),Y=new T({props:{code:"aW1wb3J0JTIwcmUlMEFmcm9tJTIwZGF0YXNldHMlMjBpbXBvcnQlMjBsb2FkX2RhdGFzZXQlMkMlMjBEYXRhc2V0JTBBJTBBJTBBJTIzJTIwSGVscGVyJTIwZnVuY3Rpb25zJTIwdG8lMjBleHRyYWN0JTIwYW5zd2VycyUyMGZyb20lMjBkaWZmZXJlbnQlMjBmb3JtYXRzJTBBZGVmJTIwZXh0cmFjdF94bWxfYW5zd2VyKHRleHQlM0ElMjBzdHIpJTIwLSUzRSUyMHN0ciUzQSUwQSUyMCUyMCUyMCUyMGFuc3dlciUyMCUzRCUyMHRleHQuc3BsaXQoJTIyJTNDYW5zd2VyJTNFJTIyKSU1Qi0xJTVEJTBBJTIwJTIwJTIwJTIwYW5zd2VyJTIwJTNEJTIwYW5zd2VyLnNwbGl0KCUyMiUzQyUyRmFuc3dlciUzRSUyMiklNUIwJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwYW5zd2VyLnN0cmlwKCklMEElMEElMEFkZWYlMjBleHRyYWN0X2hhc2hfYW5zd2VyKHRleHQlM0ElMjBzdHIpJTIwLSUzRSUyMHN0ciUyMCU3QyUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjBpZiUyMCUyMiUyMyUyMyUyMyUyMyUyMiUyMG5vdCUyMGluJTIwdGV4dCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyME5vbmUlMEElMjAlMjAlMjAlMjByZXR1cm4lMjB0ZXh0LnNwbGl0KCUyMiUyMyUyMyUyMyUyMyUyMiklNUIxJTVELnN0cmlwKCklMEElMEElMEElMjMlMjBGdW5jdGlvbiUyMHRvJTIwcHJlcGFyZSUyMHRoZSUyMEdTTThLJTIwZGF0YXNldCUwQWRlZiUyMGdldF9nc204a19xdWVzdGlvbnMoc3BsaXQlM0QlMjJ0cmFpbiUyMiklMjAtJTNFJTIwRGF0YXNldCUzQSUwQSUyMCUyMCUyMCUyMGRhdGElMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyb3BlbmFpJTJGZ3NtOGslMjIlMkMlMjAlMjJtYWluJTIyKSU1QnNwbGl0JTVEJTBBJTIwJTIwJTIwJTIwZGF0YSUyMCUzRCUyMGRhdGEubWFwKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhbWJkYSUyMHglM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwcm9tcHQlMjIlM0ElMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJyb2xlJTIyJTNBJTIwJTIyc3lzdGVtJTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMFNZU1RFTV9QUk9NUFQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjB4JTVCJTIycXVlc3Rpb24lMjIlNUQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJhbnN3ZXIlMjIlM0ElMjBleHRyYWN0X2hhc2hfYW5zd2VyKHglNUIlMjJhbnN3ZXIlMjIlNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMGRhdGElMEElMEElMEFkYXRhc2V0JTIwJTNEJTIwZ2V0X2dzbThrX3F1ZXN0aW9ucygp",highlighted:`<span class="hljs-keyword">import</span> re
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Dataset


<span class="hljs-comment"># Helper functions to extract answers from different formats</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">extract_xml_answer</span>(<span class="hljs-params">text: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span>:
    answer = text.split(<span class="hljs-string">&quot;&lt;answer&gt;&quot;</span>)[-<span class="hljs-number">1</span>]
    answer = answer.split(<span class="hljs-string">&quot;&lt;/answer&gt;&quot;</span>)[<span class="hljs-number">0</span>]
    <span class="hljs-keyword">return</span> answer.strip()


<span class="hljs-keyword">def</span> <span class="hljs-title function_">extract_hash_answer</span>(<span class="hljs-params">text: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span> | <span class="hljs-literal">None</span>:
    <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;####&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> text:
        <span class="hljs-keyword">return</span> <span class="hljs-literal">None</span>
    <span class="hljs-keyword">return</span> text.split(<span class="hljs-string">&quot;####&quot;</span>)[<span class="hljs-number">1</span>].strip()


<span class="hljs-comment"># Function to prepare the GSM8K dataset</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_gsm8k_questions</span>(<span class="hljs-params">split=<span class="hljs-string">&quot;train&quot;</span></span>) -&gt; Dataset:
    data = load_dataset(<span class="hljs-string">&quot;openai/gsm8k&quot;</span>, <span class="hljs-string">&quot;main&quot;</span>)[split]
    data = data.<span class="hljs-built_in">map</span>(
        <span class="hljs-keyword">lambda</span> x: {
            <span class="hljs-string">&quot;prompt&quot;</span>: [
                {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: SYSTEM_PROMPT},
                {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: x[<span class="hljs-string">&quot;question&quot;</span>]},
            ],
            <span class="hljs-string">&quot;answer&quot;</span>: extract_hash_answer(x[<span class="hljs-string">&quot;answer&quot;</span>]),
        }
    )
    <span class="hljs-keyword">return</span> data


dataset = get_gsm8k_questions()`,wrap:!1}}),z=new h({props:{title:"Defining Reward Functions",local:"defining-reward-functions",headingTag:"h2"}}),S=new T({props:{code:"JTIzJTIwUmV3YXJkJTIwZnVuY3Rpb24lMjB0aGF0JTIwY2hlY2tzJTIwaWYlMjB0aGUlMjBhbnN3ZXIlMjBpcyUyMGNvcnJlY3QlMEFkZWYlMjBjb3JyZWN0bmVzc19yZXdhcmRfZnVuYyhwcm9tcHRzJTJDJTIwY29tcGxldGlvbnMlMkMlMjBhbnN3ZXIlMkMlMjAqKmt3YXJncyklMjAtJTNFJTIwbGlzdCU1QmZsb2F0JTVEJTNBJTBBJTIwJTIwJTIwJTIwcmVzcG9uc2VzJTIwJTNEJTIwJTVCY29tcGxldGlvbiU1QjAlNUQlNUIlMjJjb250ZW50JTIyJTVEJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMjAlMjAlMjAlMjBxJTIwJTNEJTIwcHJvbXB0cyU1QjAlNUQlNUItMSU1RCU1QiUyMmNvbnRlbnQlMjIlNUQlMEElMjAlMjAlMjAlMjBleHRyYWN0ZWRfcmVzcG9uc2VzJTIwJTNEJTIwJTVCZXh0cmFjdF94bWxfYW5zd2VyKHIpJTIwZm9yJTIwciUyMGluJTIwcmVzcG9uc2VzJTVEJTBBJTIwJTIwJTIwJTIwcHJpbnQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyLSUyMiUyMColMjAyMCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYlMjJRdWVzdGlvbiUzQSU1Q24lN0JxJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZiUyMiU1Q25BbnN3ZXIlM0ElNUNuJTdCYW5zd2VyJTVCMCU1RCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYlMjIlNUNuUmVzcG9uc2UlM0ElNUNuJTdCcmVzcG9uc2VzJTVCMCU1RCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYlMjIlNUNuRXh0cmFjdGVkJTNBJTVDbiU3QmV4dHJhY3RlZF9yZXNwb25zZXMlNUIwJTVEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QjIuMCUyMGlmJTIwciUyMCUzRCUzRCUyMGElMjBlbHNlJTIwMC4wJTIwZm9yJTIwciUyQyUyMGElMjBpbiUyMHppcChleHRyYWN0ZWRfcmVzcG9uc2VzJTJDJTIwYW5zd2VyKSU1RCUwQSUwQSUwQSUyMyUyMFJld2FyZCUyMGZ1bmN0aW9uJTIwdGhhdCUyMGNoZWNrcyUyMGlmJTIwdGhlJTIwYW5zd2VyJTIwaXMlMjBhbiUyMGludGVnZXIlMEFkZWYlMjBpbnRfcmV3YXJkX2Z1bmMoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklMjAtJTNFJTIwbGlzdCU1QmZsb2F0JTVEJTNBJTBBJTIwJTIwJTIwJTIwcmVzcG9uc2VzJTIwJTNEJTIwJTVCY29tcGxldGlvbiU1QjAlNUQlNUIlMjJjb250ZW50JTIyJTVEJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMjAlMjAlMjAlMjBleHRyYWN0ZWRfcmVzcG9uc2VzJTIwJTNEJTIwJTVCZXh0cmFjdF94bWxfYW5zd2VyKHIpJTIwZm9yJTIwciUyMGluJTIwcmVzcG9uc2VzJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTVCMC41JTIwaWYlMjByLmlzZGlnaXQoKSUyMGVsc2UlMjAwLjAlMjBmb3IlMjByJTIwaW4lMjBleHRyYWN0ZWRfcmVzcG9uc2VzJTVEJTBBJTBBJTBBJTIzJTIwUmV3YXJkJTIwZnVuY3Rpb24lMjB0aGF0JTIwY2hlY2tzJTIwaWYlMjB0aGUlMjBjb21wbGV0aW9uJTIwZm9sbG93cyUyMHRoZSUyMHN0cmljdCUyMGZvcm1hdCUwQWRlZiUyMHN0cmljdF9mb3JtYXRfcmV3YXJkX2Z1bmMoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklMjAtJTNFJTIwbGlzdCU1QmZsb2F0JTVEJTNBJTBBJTIwJTIwJTIwJTIwcGF0dGVybiUyMCUzRCUyMHIlMjIlNUUlM0NyZWFzb25pbmclM0UlNUNuLiolM0YlNUNuJTNDJTJGcmVhc29uaW5nJTNFJTVDbiUzQ2Fuc3dlciUzRSU1Q24uKiUzRiU1Q24lM0MlMkZhbnN3ZXIlM0UlNUNuJTI0JTIyJTBBJTIwJTIwJTIwJTIwcmVzcG9uc2VzJTIwJTNEJTIwJTVCY29tcGxldGlvbiU1QjAlNUQlNUIlMjJjb250ZW50JTIyJTVEJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMjAlMjAlMjAlMjBtYXRjaGVzJTIwJTNEJTIwJTVCcmUubWF0Y2gocGF0dGVybiUyQyUyMHIpJTIwZm9yJTIwciUyMGluJTIwcmVzcG9uc2VzJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTVCMC41JTIwaWYlMjBtYXRjaCUyMGVsc2UlMjAwLjAlMjBmb3IlMjBtYXRjaCUyMGluJTIwbWF0Y2hlcyU1RCUwQSUwQSUwQSUyMyUyMFJld2FyZCUyMGZ1bmN0aW9uJTIwdGhhdCUyMGNoZWNrcyUyMGlmJTIwdGhlJTIwY29tcGxldGlvbiUyMGZvbGxvd3MlMjBhJTIwbW9yZSUyMHJlbGF4ZWQlMjBmb3JtYXQlMEFkZWYlMjBzb2Z0X2Zvcm1hdF9yZXdhcmRfZnVuYyhjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUyMC0lM0UlMjBsaXN0JTVCZmxvYXQlNUQlM0ElMEElMjAlMjAlMjAlMjBwYXR0ZXJuJTIwJTNEJTIwciUyMiUzQ3JlYXNvbmluZyUzRS4qJTNGJTNDJTJGcmVhc29uaW5nJTNFJTVDcyolM0NhbnN3ZXIlM0UuKiUzRiUzQyUyRmFuc3dlciUzRSUyMiUwQSUyMCUyMCUyMCUyMHJlc3BvbnNlcyUyMCUzRCUyMCU1QmNvbXBsZXRpb24lNUIwJTVEJTVCJTIyY29udGVudCUyMiU1RCUyMGZvciUyMGNvbXBsZXRpb24lMjBpbiUyMGNvbXBsZXRpb25zJTVEJTBBJTIwJTIwJTIwJTIwbWF0Y2hlcyUyMCUzRCUyMCU1QnJlLm1hdGNoKHBhdHRlcm4lMkMlMjByKSUyMGZvciUyMHIlMjBpbiUyMHJlc3BvbnNlcyU1RCUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QjAuNSUyMGlmJTIwbWF0Y2glMjBlbHNlJTIwMC4wJTIwZm9yJTIwbWF0Y2glMjBpbiUyMG1hdGNoZXMlNUQlMEElMEElMEElMjMlMjBSZXdhcmQlMjBmdW5jdGlvbiUyMHRoYXQlMjBjb3VudHMlMjBYTUwlMjB0YWdzJTIwYW5kJTIwcGVuYWxpemVzJTIwZXh0cmElMjBjb250ZW50JTBBZGVmJTIwY291bnRfeG1sKHRleHQpJTIwLSUzRSUyMGZsb2F0JTNBJTBBJTIwJTIwJTIwJTIwY291bnQlMjAlM0QlMjAwLjAlMEElMjAlMjAlMjAlMjBpZiUyMHRleHQuY291bnQoJTIyJTNDcmVhc29uaW5nJTNFJTVDbiUyMiklMjAlM0QlM0QlMjAxJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY291bnQlMjAlMkIlM0QlMjAwLjEyNSUwQSUyMCUyMCUyMCUyMGlmJTIwdGV4dC5jb3VudCglMjIlNUNuJTNDJTJGcmVhc29uaW5nJTNFJTVDbiUyMiklMjAlM0QlM0QlMjAxJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY291bnQlMjAlMkIlM0QlMjAwLjEyNSUwQSUyMCUyMCUyMCUyMGlmJTIwdGV4dC5jb3VudCglMjIlNUNuJTNDYW5zd2VyJTNFJTVDbiUyMiklMjAlM0QlM0QlMjAxJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY291bnQlMjAlMkIlM0QlMjAwLjEyNSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvdW50JTIwLSUzRCUyMGxlbih0ZXh0LnNwbGl0KCUyMiU1Q24lM0MlMkZhbnN3ZXIlM0UlNUNuJTIyKSU1Qi0xJTVEKSUyMColMjAwLjAwMSUwQSUyMCUyMCUyMCUyMGlmJTIwdGV4dC5jb3VudCglMjIlNUNuJTNDJTJGYW5zd2VyJTNFJTIyKSUyMCUzRCUzRCUyMDElM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb3VudCUyMCUyQiUzRCUyMDAuMTI1JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY291bnQlMjAtJTNEJTIwKGxlbih0ZXh0LnNwbGl0KCUyMiU1Q24lM0MlMkZhbnN3ZXIlM0UlMjIpJTVCLTElNUQpJTIwLSUyMDEpJTIwKiUyMDAuMDAxJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwY291bnQlMEElMEElMEFkZWYlMjB4bWxjb3VudF9yZXdhcmRfZnVuYyhjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUyMC0lM0UlMjBsaXN0JTVCZmxvYXQlNUQlM0ElMEElMjAlMjAlMjAlMjBjb250ZW50cyUyMCUzRCUyMCU1QmNvbXBsZXRpb24lNUIwJTVEJTVCJTIyY29udGVudCUyMiU1RCUyMGZvciUyMGNvbXBsZXRpb24lMjBpbiUyMGNvbXBsZXRpb25zJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTVCY291bnRfeG1sKGMpJTIwZm9yJTIwYyUyMGluJTIwY29udGVudHMlNUQ=",highlighted:`<span class="hljs-comment"># Reward function that checks if the answer is correct</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">correctness_reward_func</span>(<span class="hljs-params">prompts, completions, answer, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
    responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
    q = prompts[<span class="hljs-number">0</span>][-<span class="hljs-number">1</span>][<span class="hljs-string">&quot;content&quot;</span>]
    extracted_responses = [extract_xml_answer(r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
    <span class="hljs-built_in">print</span>(
        <span class="hljs-string">&quot;-&quot;</span> * <span class="hljs-number">20</span>,
        <span class="hljs-string">f&quot;Question:\\n<span class="hljs-subst">{q}</span>&quot;</span>,
        <span class="hljs-string">f&quot;\\nAnswer:\\n<span class="hljs-subst">{answer[<span class="hljs-number">0</span>]}</span>&quot;</span>,
        <span class="hljs-string">f&quot;\\nResponse:\\n<span class="hljs-subst">{responses[<span class="hljs-number">0</span>]}</span>&quot;</span>,
        <span class="hljs-string">f&quot;\\nExtracted:\\n<span class="hljs-subst">{extracted_responses[<span class="hljs-number">0</span>]}</span>&quot;</span>,
    )
    <span class="hljs-keyword">return</span> [<span class="hljs-number">2.0</span> <span class="hljs-keyword">if</span> r == a <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> r, a <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(extracted_responses, answer)]


<span class="hljs-comment"># Reward function that checks if the answer is an integer</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">int_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
    responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
    extracted_responses = [extract_xml_answer(r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
    <span class="hljs-keyword">return</span> [<span class="hljs-number">0.5</span> <span class="hljs-keyword">if</span> r.isdigit() <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> extracted_responses]


<span class="hljs-comment"># Reward function that checks if the completion follows the strict format</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">strict_format_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
    pattern = <span class="hljs-string">r&quot;^&lt;reasoning&gt;\\n.*?\\n&lt;/reasoning&gt;\\n&lt;answer&gt;\\n.*?\\n&lt;/answer&gt;\\n$&quot;</span>
    responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
    matches = [re.<span class="hljs-keyword">match</span>(pattern, r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
    <span class="hljs-keyword">return</span> [<span class="hljs-number">0.5</span> <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">in</span> matches]


<span class="hljs-comment"># Reward function that checks if the completion follows a more relaxed format</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">soft_format_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
    pattern = <span class="hljs-string">r&quot;&lt;reasoning&gt;.*?&lt;/reasoning&gt;\\s*&lt;answer&gt;.*?&lt;/answer&gt;&quot;</span>
    responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
    matches = [re.<span class="hljs-keyword">match</span>(pattern, r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
    <span class="hljs-keyword">return</span> [<span class="hljs-number">0.5</span> <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">in</span> matches]


<span class="hljs-comment"># Reward function that counts XML tags and penalizes extra content</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">count_xml</span>(<span class="hljs-params">text</span>) -&gt; <span class="hljs-built_in">float</span>:
    count = <span class="hljs-number">0.0</span>
    <span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;&lt;reasoning&gt;\\n&quot;</span>) == <span class="hljs-number">1</span>:
        count += <span class="hljs-number">0.125</span>
    <span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;\\n&lt;/reasoning&gt;\\n&quot;</span>) == <span class="hljs-number">1</span>:
        count += <span class="hljs-number">0.125</span>
    <span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;\\n&lt;answer&gt;\\n&quot;</span>) == <span class="hljs-number">1</span>:
        count += <span class="hljs-number">0.125</span>
        count -= <span class="hljs-built_in">len</span>(text.split(<span class="hljs-string">&quot;\\n&lt;/answer&gt;\\n&quot;</span>)[-<span class="hljs-number">1</span>]) * <span class="hljs-number">0.001</span>
    <span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;\\n&lt;/answer&gt;&quot;</span>) == <span class="hljs-number">1</span>:
        count += <span class="hljs-number">0.125</span>
        count -= (<span class="hljs-built_in">len</span>(text.split(<span class="hljs-string">&quot;\\n&lt;/answer&gt;&quot;</span>)[-<span class="hljs-number">1</span>]) - <span class="hljs-number">1</span>) * <span class="hljs-number">0.001</span>
    <span class="hljs-keyword">return</span> count


<span class="hljs-keyword">def</span> <span class="hljs-title function_">xmlcount_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
    contents = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
    <span class="hljs-keyword">return</span> [count_xml(c) <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> contents]`,wrap:!1}}),P=new h({props:{title:"Training with GRPO",local:"training-with-grpo",headingTag:"h2"}}),O=new T({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMkMlMjBHUlBPVHJhaW5lciUwQSUwQW1heF9wcm9tcHRfbGVuZ3RoJTIwJTNEJTIwMjU2JTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwbGVhcm5pbmdfcmF0ZSUzRDVlLTYlMkMlMEElMjAlMjAlMjAlMjBhZGFtX2JldGExJTNEMC45JTJDJTBBJTIwJTIwJTIwJTIwYWRhbV9iZXRhMiUzRDAuOTklMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRfZGVjYXklM0QwLjElMkMlMEElMjAlMjAlMjAlMjB3YXJtdXBfcmF0aW8lM0QwLjElMkMlMEElMjAlMjAlMjAlMjBscl9zY2hlZHVsZXJfdHlwZSUzRCUyMmNvc2luZSUyMiUyQyUwQSUyMCUyMCUyMCUyMG9wdGltJTNEJTIycGFnZWRfYWRhbXdfOGJpdCUyMiUyQyUwQSUyMCUyMCUyMCUyMGxvZ2dpbmdfc3RlcHMlM0QxJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNEMSUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDElMkMlMjAlMjAlMjMlMjBJbmNyZWFzZSUyMHRvJTIwNCUyMGZvciUyMHNtb290aGVyJTIwdHJhaW5pbmclMEElMjAlMjAlMjAlMjBudW1fZ2VuZXJhdGlvbnMlM0Q2JTJDJTIwJTIwJTIzJTIwRGVjcmVhc2UlMjBpZiUyMG91dCUyMG9mJTIwbWVtb3J5JTBBJTIwJTIwJTIwJTIwbWF4X3Byb21wdF9sZW5ndGglM0RtYXhfcHJvbXB0X2xlbmd0aCUyQyUwQSUyMCUyMCUyMCUyMG1heF9jb21wbGV0aW9uX2xlbmd0aCUzRG1heF9zZXFfbGVuZ3RoJTIwLSUyMG1heF9wcm9tcHRfbGVuZ3RoJTJDJTBBJTIwJTIwJTIwJTIwJTIzJTIwbnVtX3RyYWluX2Vwb2NocyUyMCUzRCUyMDElMkMlMjAlMjMlMjBTZXQlMjB0byUyMDElMjBmb3IlMjBhJTIwZnVsbCUyMHRyYWluaW5nJTIwcnVuJTBBJTIwJTIwJTIwJTIwbWF4X3N0ZXBzJTNEMjUwJTJDJTBBJTIwJTIwJTIwJTIwc2F2ZV9zdGVwcyUzRDI1MCUyQyUwQSUyMCUyMCUyMCUyMG1heF9ncmFkX25vcm0lM0QwLjElMkMlMEElMjAlMjAlMjAlMjByZXBvcnRfdG8lM0QlMjJub25lJTIyJTJDJTIwJTIwJTIzJTIwQ2FuJTIwdXNlJTIwV2VpZ2h0cyUyMCUyNiUyMEJpYXNlcyUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJvdXRwdXRzJTIyJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBHUlBPVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwcHJvY2Vzc2luZ19jbGFzcyUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHhtbGNvdW50X3Jld2FyZF9mdW5jJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc29mdF9mb3JtYXRfcmV3YXJkX2Z1bmMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdHJpY3RfZm9ybWF0X3Jld2FyZF9mdW5jJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW50X3Jld2FyZF9mdW5jJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29ycmVjdG5lc3NfcmV3YXJkX2Z1bmMlMkMlMEElMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0RkYXRhc2V0JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig, GRPOTrainer

max_prompt_length = <span class="hljs-number">256</span>

training_args = GRPOConfig(
    learning_rate=<span class="hljs-number">5e-6</span>,
    adam_beta1=<span class="hljs-number">0.9</span>,
    adam_beta2=<span class="hljs-number">0.99</span>,
    weight_decay=<span class="hljs-number">0.1</span>,
    warmup_ratio=<span class="hljs-number">0.1</span>,
    lr_scheduler_type=<span class="hljs-string">&quot;cosine&quot;</span>,
    optim=<span class="hljs-string">&quot;paged_adamw_8bit&quot;</span>,
    logging_steps=<span class="hljs-number">1</span>,
    per_device_train_batch_size=<span class="hljs-number">1</span>,
    gradient_accumulation_steps=<span class="hljs-number">1</span>,  <span class="hljs-comment"># Increase to 4 for smoother training</span>
    num_generations=<span class="hljs-number">6</span>,  <span class="hljs-comment"># Decrease if out of memory</span>
    max_prompt_length=max_prompt_length,
    max_completion_length=max_seq_length - max_prompt_length,
    <span class="hljs-comment"># num_train_epochs = 1, # Set to 1 for a full training run</span>
    max_steps=<span class="hljs-number">250</span>,
    save_steps=<span class="hljs-number">250</span>,
    max_grad_norm=<span class="hljs-number">0.1</span>,
    report_to=<span class="hljs-string">&quot;none&quot;</span>,  <span class="hljs-comment"># Can use Weights &amp; Biases</span>
    output_dir=<span class="hljs-string">&quot;outputs&quot;</span>,
)

trainer = GRPOTrainer(
    model=model,
    processing_class=tokenizer,
    reward_funcs=[
        xmlcount_reward_func,
        soft_format_reward_func,
        strict_format_reward_func,
        int_reward_func,
        correctness_reward_func,
    ],
    args=training_args,
    train_dataset=dataset,
)`,wrap:!1}}),el=new T({props:{code:"dHJhaW5lci50cmFpbigp",highlighted:"trainer.train()",wrap:!1}}),f=new ks({props:{warning:!0,$$slots:{default:[Ct]},$$scope:{ctx:d}}}),nl=new h({props:{title:"Testing the Model",local:"testing-the-model",headingTag:"h2"}}),Ml=new T({props:{code:"bW9kZWwuc2F2ZV9sb3JhKCUyMmdycG9fc2F2ZWRfbG9yYSUyMik=",highlighted:'model.save_lora(<span class="hljs-string">&quot;grpo_saved_lora&quot;</span>)',wrap:!1}}),il=new T({props:{code:"ZnJvbSUyMHZsbG0lMjBpbXBvcnQlMjBTYW1wbGluZ1BhcmFtcyUwQSUwQXRleHQlMjAlM0QlMjB0b2tlbml6ZXIuYXBwbHlfY2hhdF90ZW1wbGF0ZSglMEElMjAlMjAlMjAlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJyb2xlJTIyJTNBJTIwJTIyc3lzdGVtJTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMFNZU1RFTV9QUk9NUFQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJDYWxjdWxhdGUlMjBwaS4lMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZSUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwYWRkX2dlbmVyYXRpb25fcHJvbXB0JTNEVHJ1ZSUyQyUwQSklMEElMEFzYW1wbGluZ19wYXJhbXMlMjAlM0QlMjBTYW1wbGluZ1BhcmFtcyglMEElMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzRDAuOCUyQyUwQSUyMCUyMCUyMCUyMHRvcF9wJTNEMC45NSUyQyUwQSUyMCUyMCUyMCUyMG1heF90b2tlbnMlM0QxMDI0JTJDJTBBKSUwQW91dHB1dCUyMCUzRCUyMCglMEElMjAlMjAlMjAlMjBtb2RlbC5mYXN0X2dlbmVyYXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRleHQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzYW1wbGluZ19wYXJhbXMlM0RzYW1wbGluZ19wYXJhbXMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3JhX3JlcXVlc3QlM0Rtb2RlbC5sb2FkX2xvcmEoJTIyZ3Jwb19zYXZlZF9sb3JhJTIyKSUyQyUwQSUyMCUyMCUyMCUyMCklNUIwJTVEJTBBJTIwJTIwJTIwJTIwLm91dHB1dHMlNUIwJTVEJTBBJTIwJTIwJTIwJTIwLnRleHQlMEEpJTBBJTBBcHJpbnQob3V0cHV0KQ==",highlighted:`<span class="hljs-keyword">from</span> vllm <span class="hljs-keyword">import</span> SamplingParams

text = tokenizer.apply_chat_template(
    [
        {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: SYSTEM_PROMPT},
        {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Calculate pi.&quot;</span>},
    ],
    tokenize=<span class="hljs-literal">False</span>,
    add_generation_prompt=<span class="hljs-literal">True</span>,
)

sampling_params = SamplingParams(
    temperature=<span class="hljs-number">0.8</span>,
    top_p=<span class="hljs-number">0.95</span>,
    max_tokens=<span class="hljs-number">1024</span>,
)
output = (
    model.fast_generate(
        text,
        sampling_params=sampling_params,
        lora_request=model.load_lora(<span class="hljs-string">&quot;grpo_saved_lora&quot;</span>),
    )[<span class="hljs-number">0</span>]
    .outputs[<span class="hljs-number">0</span>]
    .text
)

<span class="hljs-built_in">print</span>(output)`,wrap:!1}}),rl=new h({props:{title:"Saving the Model",local:"saving-the-model",headingTag:"h2"}}),yl=new T({props:{code:"JTIzJTIwU2F2ZSUyMHRvJTIwMTYtYml0JTIwcHJlY2lzaW9uJTBBbW9kZWwuc2F2ZV9wcmV0cmFpbmVkX21lcmdlZCglMjJtb2RlbCUyMiUyQyUyMHRva2VuaXplciUyQyUyMHNhdmVfbWV0aG9kJTNEJTIybWVyZ2VkXzE2Yml0JTIyKQ==",highlighted:`<span class="hljs-comment"># Save to 16-bit precision</span>
model.save_pretrained_merged(<span class="hljs-string">&quot;model&quot;</span>, tokenizer, save_method=<span class="hljs-string">&quot;merged_16bit&quot;</span>)`,wrap:!1}}),jl=new h({props:{title:"Pushing to Hugging Face Hub",local:"pushing-to-hugging-face-hub",headingTag:"h2"}}),Jl=new T({props:{code:"JTIzJTIwUHVzaCUyMHRvJTIwSHVnZ2luZyUyMEZhY2UlMjBIdWIlMjAocmVxdWlyZXMlMjBhJTIwdG9rZW4pJTBBbW9kZWwucHVzaF90b19odWJfbWVyZ2VkKCUwQSUyMCUyMCUyMCUyMCUyMnlvdXItdXNlcm5hbWUlMkZtb2RlbC1uYW1lJTIyJTJDJTIwdG9rZW5pemVyJTJDJTIwc2F2ZV9tZXRob2QlM0QlMjJtZXJnZWRfMTZiaXQlMjIlMkMlMjB0b2tlbiUzRCUyMnlvdXItdG9rZW4lMjIlMEEp",highlighted:`<span class="hljs-comment"># Push to Hugging Face Hub (requires a token)</span>
model.push_to_hub_merged(
    <span class="hljs-string">&quot;your-username/model-name&quot;</span>, tokenizer, save_method=<span class="hljs-string">&quot;merged_16bit&quot;</span>, token=<span class="hljs-string">&quot;your-token&quot;</span>
)`,wrap:!1}}),ul=new T({props:{code:"bW9kZWwucHVzaF90b19odWJfZ2d1ZiglMEElMjAlMjAlMjAlMjAlMjJ5b3VyLXVzZXJuYW1lJTJGbW9kZWwtbmFtZSUyMiUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9tZXRob2QlM0QlNUIlMjJxNF9rX20lMjIlMkMlMjAlMjJxOF8wJTIyJTJDJTIwJTIycTVfa19tJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW4lM0QlMjJ5b3VyLXRva2VuJTIyJTJDJTBBKQ==",highlighted:`model.push_to_hub_gguf(
    <span class="hljs-string">&quot;your-username/model-name&quot;</span>,
    tokenizer,
    quantization_method=[<span class="hljs-string">&quot;q4_k_m&quot;</span>, <span class="hljs-string">&quot;q8_0&quot;</span>, <span class="hljs-string">&quot;q5_k_m&quot;</span>],
    token=<span class="hljs-string">&quot;your-token&quot;</span>,
)`,wrap:!1}}),Ul=new h({props:{title:"Conclusion",local:"conclusion",headingTag:"h2"}}),Cl=new bt({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter12/6.mdx"}}),{c(){i=M("meta"),u=n(),m=M("p"),U=n(),r(C.$$.fragment),Bl=n(),r(g.$$.fragment),Al=n(),B=M("p"),B.innerHTML=$s,Gl=n(),A=M("p"),A.textContent=Qs,_l=n(),r(b.$$.fragment),Zl=n(),r(G.$$.fragment),kl=n(),_=M("p"),_.textContent=Es,Vl=n(),r(Z.$$.fragment),$l=n(),r(k.$$.fragment),Ql=n(),V=M("p"),V.innerHTML=Rs,El=n(),r($.$$.fragment),Rl=n(),Q=M("p"),Q.textContent=Ns,Nl=n(),r(E.$$.fragment),vl=n(),R=M("p"),R.innerHTML=vs,xl=n(),r(I.$$.fragment),Xl=n(),r(N.$$.fragment),Wl=n(),v=M("p"),v.textContent=xs,Yl=n(),x=M("p"),x.textContent=Xs,ql=n(),r(X.$$.fragment),zl=n(),W=M("p"),W.textContent=Ws,Fl=n(),r(Y.$$.fragment),Hl=n(),q=M("p"),q.textContent=Ys,Sl=n(),r(z.$$.fragment),Ll=n(),F=M("p"),F.innerHTML=qs,Dl=n(),H=M("p"),H.textContent=zs,Pl=n(),r(S.$$.fragment),Kl=n(),L=M("p"),L.textContent=Fs,Ol=n(),D=M("table"),D.innerHTML=Hs,ls=n(),r(P.$$.fragment),ss=n(),K=M("p"),K.innerHTML=Ss,ts=n(),r(O.$$.fragment),es=n(),ll=M("p"),ll.innerHTML=Ls,ns=n(),sl=M("ul"),sl.innerHTML=Ds,as=n(),tl=M("p"),tl.textContent=Ps,Ms=n(),r(el.$$.fragment),ps=n(),r(f.$$.fragment),is=n(),r(nl.$$.fragment),os=n(),al=M("p"),al.textContent=Ks,rs=n(),r(Ml.$$.fragment),cs=n(),pl=M("p"),pl.textContent=Os,ys=n(),r(il.$$.fragment),js=n(),ol=M("p"),ol.textContent=lt,ws=n(),r(rl.$$.fragment),Js=n(),cl=M("p"),cl.textContent=st,ms=n(),r(yl.$$.fragment),us=n(),r(jl.$$.fragment),Ts=n(),wl=M("p"),wl.innerHTML=tt,Us=n(),r(Jl.$$.fragment),hs=n(),ml=M("p"),ml.textContent=et,ds=n(),r(ul.$$.fragment),bs=n(),Tl=M("p"),Tl.textContent=nt,Is=n(),r(Ul.$$.fragment),fs=n(),hl=M("p"),hl.textContent=at,Cs=n(),dl=M("ol"),dl.innerHTML=Mt,gs=n(),bl=M("p"),bl.textContent=pt,Bs=n(),Il=M("p"),Il.textContent=it,As=n(),fl=M("ul"),fl.innerHTML=ot,Gs=n(),r(Cl.$$.fragment),_s=n(),gl=M("p"),this.h()},l(l){const s=Ut("svelte-u9bgzb",document.head);i=p(s,"META",{name:!0,content:!0}),s.forEach(t),u=a(l),m=p(l,"P",{}),jt(m).forEach(t),U=a(l),c(C.$$.fragment,l),Bl=a(l),c(g.$$.fragment,l),Al=a(l),B=p(l,"P",{"data-svelte-h":!0}),o(B)!=="svelte-3dwsf2"&&(B.innerHTML=$s),Gl=a(l),A=p(l,"P",{"data-svelte-h":!0}),o(A)!=="svelte-v44mtq"&&(A.textContent=Qs),_l=a(l),c(b.$$.fragment,l),Zl=a(l),c(G.$$.fragment,l),kl=a(l),_=p(l,"P",{"data-svelte-h":!0}),o(_)!=="svelte-7apc0w"&&(_.textContent=Es),Vl=a(l),c(Z.$$.fragment,l),$l=a(l),c(k.$$.fragment,l),Ql=a(l),V=p(l,"P",{"data-svelte-h":!0}),o(V)!=="svelte-uhz6q0"&&(V.innerHTML=Rs),El=a(l),c($.$$.fragment,l),Rl=a(l),Q=p(l,"P",{"data-svelte-h":!0}),o(Q)!=="svelte-1o52er5"&&(Q.textContent=Ns),Nl=a(l),c(E.$$.fragment,l),vl=a(l),R=p(l,"P",{"data-svelte-h":!0}),o(R)!=="svelte-ihivum"&&(R.innerHTML=vs),xl=a(l),c(I.$$.fragment,l),Xl=a(l),c(N.$$.fragment,l),Wl=a(l),v=p(l,"P",{"data-svelte-h":!0}),o(v)!=="svelte-1jqnhvo"&&(v.textContent=xs),Yl=a(l),x=p(l,"P",{"data-svelte-h":!0}),o(x)!=="svelte-222xwa"&&(x.textContent=Xs),ql=a(l),c(X.$$.fragment,l),zl=a(l),W=p(l,"P",{"data-svelte-h":!0}),o(W)!=="svelte-uuip11"&&(W.textContent=Ws),Fl=a(l),c(Y.$$.fragment,l),Hl=a(l),q=p(l,"P",{"data-svelte-h":!0}),o(q)!=="svelte-yuvu2n"&&(q.textContent=Ys),Sl=a(l),c(z.$$.fragment,l),Ll=a(l),F=p(l,"P",{"data-svelte-h":!0}),o(F)!=="svelte-tq7f7j"&&(F.innerHTML=qs),Dl=a(l),H=p(l,"P",{"data-svelte-h":!0}),o(H)!=="svelte-1bbmyq2"&&(H.textContent=zs),Pl=a(l),c(S.$$.fragment,l),Kl=a(l),L=p(l,"P",{"data-svelte-h":!0}),o(L)!=="svelte-10lwv8u"&&(L.textContent=Fs),Ol=a(l),D=p(l,"TABLE",{"data-svelte-h":!0}),o(D)!=="svelte-beg5py"&&(D.innerHTML=Hs),ls=a(l),c(P.$$.fragment,l),ss=a(l),K=p(l,"P",{"data-svelte-h":!0}),o(K)!=="svelte-9dkvch"&&(K.innerHTML=Ss),ts=a(l),c(O.$$.fragment,l),es=a(l),ll=p(l,"P",{"data-svelte-h":!0}),o(ll)!=="svelte-1jo17wb"&&(ll.innerHTML=Ls),ns=a(l),sl=p(l,"UL",{"data-svelte-h":!0}),o(sl)!=="svelte-eo3csz"&&(sl.innerHTML=Ds),as=a(l),tl=p(l,"P",{"data-svelte-h":!0}),o(tl)!=="svelte-c65a8i"&&(tl.textContent=Ps),Ms=a(l),c(el.$$.fragment,l),ps=a(l),c(f.$$.fragment,l),is=a(l),c(nl.$$.fragment,l),os=a(l),al=p(l,"P",{"data-svelte-h":!0}),o(al)!=="svelte-dwjtn"&&(al.textContent=Ks),rs=a(l),c(Ml.$$.fragment,l),cs=a(l),pl=p(l,"P",{"data-svelte-h":!0}),o(pl)!=="svelte-6mnmyk"&&(pl.textContent=Os),ys=a(l),c(il.$$.fragment,l),js=a(l),ol=p(l,"P",{"data-svelte-h":!0}),o(ol)!=="svelte-n6ify3"&&(ol.textContent=lt),ws=a(l),c(rl.$$.fragment,l),Js=a(l),cl=p(l,"P",{"data-svelte-h":!0}),o(cl)!=="svelte-8mgtgx"&&(cl.textContent=st),ms=a(l),c(yl.$$.fragment,l),us=a(l),c(jl.$$.fragment,l),Ts=a(l),wl=p(l,"P",{"data-svelte-h":!0}),o(wl)!=="svelte-5e9stk"&&(wl.innerHTML=tt),Us=a(l),c(Jl.$$.fragment,l),hs=a(l),ml=p(l,"P",{"data-svelte-h":!0}),o(ml)!=="svelte-hwdwrb"&&(ml.textContent=et),ds=a(l),c(ul.$$.fragment,l),bs=a(l),Tl=p(l,"P",{"data-svelte-h":!0}),o(Tl)!=="svelte-102ov0f"&&(Tl.textContent=nt),Is=a(l),c(Ul.$$.fragment,l),fs=a(l),hl=p(l,"P",{"data-svelte-h":!0}),o(hl)!=="svelte-onfwxy"&&(hl.textContent=at),Cs=a(l),dl=p(l,"OL",{"data-svelte-h":!0}),o(dl)!=="svelte-j3tqlf"&&(dl.innerHTML=Mt),gs=a(l),bl=p(l,"P",{"data-svelte-h":!0}),o(bl)!=="svelte-ntlp2y"&&(bl.textContent=pt),Bs=a(l),Il=p(l,"P",{"data-svelte-h":!0}),o(Il)!=="svelte-1r9yves"&&(Il.textContent=it),As=a(l),fl=p(l,"UL",{"data-svelte-h":!0}),o(fl)!=="svelte-1fz95oy"&&(fl.innerHTML=ot),Gs=a(l),c(Cl.$$.fragment,l),_s=a(l),gl=p(l,"P",{}),jt(gl).forEach(t),this.h()},h(){wt(i,"name","hf:doc:metadata"),wt(i,"content",Bt)},m(l,s){ht(document.head,i),e(l,u,s),e(l,m,s),e(l,U,s),y(C,l,s),e(l,Bl,s),y(g,l,s),e(l,Al,s),e(l,B,s),e(l,Gl,s),e(l,A,s),e(l,_l,s),y(b,l,s),e(l,Zl,s),y(G,l,s),e(l,kl,s),e(l,_,s),e(l,Vl,s),y(Z,l,s),e(l,$l,s),y(k,l,s),e(l,Ql,s),e(l,V,s),e(l,El,s),y($,l,s),e(l,Rl,s),e(l,Q,s),e(l,Nl,s),y(E,l,s),e(l,vl,s),e(l,R,s),e(l,xl,s),y(I,l,s),e(l,Xl,s),y(N,l,s),e(l,Wl,s),e(l,v,s),e(l,Yl,s),e(l,x,s),e(l,ql,s),y(X,l,s),e(l,zl,s),e(l,W,s),e(l,Fl,s),y(Y,l,s),e(l,Hl,s),e(l,q,s),e(l,Sl,s),y(z,l,s),e(l,Ll,s),e(l,F,s),e(l,Dl,s),e(l,H,s),e(l,Pl,s),y(S,l,s),e(l,Kl,s),e(l,L,s),e(l,Ol,s),e(l,D,s),e(l,ls,s),y(P,l,s),e(l,ss,s),e(l,K,s),e(l,ts,s),y(O,l,s),e(l,es,s),e(l,ll,s),e(l,ns,s),e(l,sl,s),e(l,as,s),e(l,tl,s),e(l,Ms,s),y(el,l,s),e(l,ps,s),y(f,l,s),e(l,is,s),y(nl,l,s),e(l,os,s),e(l,al,s),e(l,rs,s),y(Ml,l,s),e(l,cs,s),e(l,pl,s),e(l,ys,s),y(il,l,s),e(l,js,s),e(l,ol,s),e(l,ws,s),y(rl,l,s),e(l,Js,s),e(l,cl,s),e(l,ms,s),y(yl,l,s),e(l,us,s),y(jl,l,s),e(l,Ts,s),e(l,wl,s),e(l,Us,s),y(Jl,l,s),e(l,hs,s),e(l,ml,s),e(l,ds,s),y(ul,l,s),e(l,bs,s),e(l,Tl,s),e(l,Is,s),y(Ul,l,s),e(l,fs,s),e(l,hl,s),e(l,Cs,s),e(l,dl,s),e(l,gs,s),e(l,bl,s),e(l,Bs,s),e(l,Il,s),e(l,As,s),e(l,fl,s),e(l,Gs,s),y(Cl,l,s),e(l,_s,s),e(l,gl,s),Zs=!0},p(l,[s]){const rt={};s&2&&(rt.$$scope={dirty:s,ctx:l}),b.$set(rt);const ct={};s&2&&(ct.$$scope={dirty:s,ctx:l}),I.$set(ct);const yt={};s&2&&(yt.$$scope={dirty:s,ctx:l}),f.$set(yt)},i(l){Zs||(j(C.$$.fragment,l),j(g.$$.fragment,l),j(b.$$.fragment,l),j(G.$$.fragment,l),j(Z.$$.fragment,l),j(k.$$.fragment,l),j($.$$.fragment,l),j(E.$$.fragment,l),j(I.$$.fragment,l),j(N.$$.fragment,l),j(X.$$.fragment,l),j(Y.$$.fragment,l),j(z.$$.fragment,l),j(S.$$.fragment,l),j(P.$$.fragment,l),j(O.$$.fragment,l),j(el.$$.fragment,l),j(f.$$.fragment,l),j(nl.$$.fragment,l),j(Ml.$$.fragment,l),j(il.$$.fragment,l),j(rl.$$.fragment,l),j(yl.$$.fragment,l),j(jl.$$.fragment,l),j(Jl.$$.fragment,l),j(ul.$$.fragment,l),j(Ul.$$.fragment,l),j(Cl.$$.fragment,l),Zs=!0)},o(l){w(C.$$.fragment,l),w(g.$$.fragment,l),w(b.$$.fragment,l),w(G.$$.fragment,l),w(Z.$$.fragment,l),w(k.$$.fragment,l),w($.$$.fragment,l),w(E.$$.fragment,l),w(I.$$.fragment,l),w(N.$$.fragment,l),w(X.$$.fragment,l),w(Y.$$.fragment,l),w(z.$$.fragment,l),w(S.$$.fragment,l),w(P.$$.fragment,l),w(O.$$.fragment,l),w(el.$$.fragment,l),w(f.$$.fragment,l),w(nl.$$.fragment,l),w(Ml.$$.fragment,l),w(il.$$.fragment,l),w(rl.$$.fragment,l),w(yl.$$.fragment,l),w(jl.$$.fragment,l),w(Jl.$$.fragment,l),w(ul.$$.fragment,l),w(Ul.$$.fragment,l),w(Cl.$$.fragment,l),Zs=!1},d(l){l&&(t(u),t(m),t(U),t(Bl),t(Al),t(B),t(Gl),t(A),t(_l),t(Zl),t(kl),t(_),t(Vl),t($l),t(Ql),t(V),t(El),t(Rl),t(Q),t(Nl),t(vl),t(R),t(xl),t(Xl),t(Wl),t(v),t(Yl),t(x),t(ql),t(zl),t(W),t(Fl),t(Hl),t(q),t(Sl),t(Ll),t(F),t(Dl),t(H),t(Pl),t(Kl),t(L),t(Ol),t(D),t(ls),t(ss),t(K),t(ts),t(es),t(ll),t(ns),t(sl),t(as),t(tl),t(Ms),t(ps),t(is),t(os),t(al),t(rs),t(cs),t(pl),t(ys),t(js),t(ol),t(ws),t(Js),t(cl),t(ms),t(us),t(Ts),t(wl),t(Us),t(hs),t(ml),t(ds),t(bs),t(Tl),t(Is),t(fs),t(hl),t(Cs),t(dl),t(gs),t(bl),t(Bs),t(Il),t(As),t(fl),t(Gs),t(_s),t(gl)),t(i),J(C,l),J(g,l),J(b,l),J(G,l),J(Z,l),J(k,l),J($,l),J(E,l),J(I,l),J(N,l),J(X,l),J(Y,l),J(z,l),J(S,l),J(P,l),J(O,l),J(el,l),J(f,l),J(nl,l),J(Ml,l),J(il,l),J(rl,l),J(yl,l),J(jl,l),J(Jl,l),J(ul,l),J(Ul,l),J(Cl,l)}}}const Bt='{"title":"Practical Exercise: GRPO with Unsloth","local":"practical-exercise-grpo-with-unsloth","sections":[{"title":"Install dependencies","local":"install-dependencies","sections":[],"depth":2},{"title":"Setting up Unsloth","local":"setting-up-unsloth","sections":[],"depth":2},{"title":"Data Preparation","local":"data-preparation","sections":[],"depth":2},{"title":"Defining Reward Functions","local":"defining-reward-functions","sections":[],"depth":2},{"title":"Training with GRPO","local":"training-with-grpo","sections":[],"depth":2},{"title":"Testing the Model","local":"testing-the-model","sections":[],"depth":2},{"title":"Saving the Model","local":"saving-the-model","sections":[],"depth":2},{"title":"Pushing to Hugging Face Hub","local":"pushing-to-hugging-face-hub","sections":[],"depth":2},{"title":"Conclusion","local":"conclusion","sections":[],"depth":2}],"depth":1}';function At(d){return mt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Qt extends ut{constructor(i){super(),Tt(this,i,At,gt,Jt,{})}}export{Qt as component};