openenv/pr_751/en/_app/immutable/nodes/63.00a5b6cf.js

import{s as Xe,n as Se,o as Fe}from"../chunks/scheduler.2b22cead.js";import{S as qe,i as ze,e as M,s as n,c as i,h as Ye,a as p,d as e,b as a,f as $e,g as o,j as r,k as pe,l as xe,m as t,n as y,t as c,o as j,p as U}from"../chunks/index.1a0e8013.js";import{C as He,H as u,E as Le}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.a01de1eb.js";import{C as m}from"../chunks/CodeBlock.81c04d9c.js";function Pe(ie){let J,Bl,Cl,Zl,w,El,d,Gl,h,oe='<a href="https://colab.research.google.com/github/huggingface/OpenEnv/blob/main/examples/sft_warmup.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>',vl,b,re=`OpenEnv environments are not only useful for RL training — they are also a natural tool for <strong>collecting
rollouts that become supervised training data</strong>. The environment handles episode management, automatic scoring,
and reproducibility, so you get a reward-labeled dataset without writing any of that infrastructure yourself.`,Rl,I,ye="This tutorial shows the full pipeline:",kl,g,ce="<li>Run a strong teacher model inside an OpenEnv environment to collect rollouts.</li> <li>Use the environment’s reward signal to filter out incorrect examples automatically.</li> <li>Train a smaller student model on the filtered rollouts with TRL’s <code>SFTTrainer</code>.</li>",Ql,f,je=`As a concrete application, the resulting checkpoint is used as a warm-start for GRPO: once the student
reliably produces valid tool calls, GRPO’s <code>reward_std</code> is non-zero from the first batch and the reward
curve climbs immediately.`,Nl,C,_l,A,Ue=`Building a supervised dataset usually means writing a custom collection loop, a scorer, and episode
bookkeeping. An OpenEnv environment gives you all three out of the box:`,Vl,B,me=`<li><strong>Automatic scoring</strong> — every <code>step()</code> returns a reward. Filter by <code>reward == 1.0</code> and you have a
clean, correct dataset with no manual labelling.</li> <li><strong>Reproducible episodes</strong> — <code>reset(seed=42, size=N)</code> produces the same sequence of problems every
run. Anyone can regenerate the exact dataset.</li> <li><strong>Configurable difficulty</strong> — adjust <code>DATASET_CONFIG</code> to control problem complexity without changing
any collection code.</li> <li><strong>Portable across environments</strong> — the same collect → filter → train pipeline works for any OpenEnv
environment. Swap the env and the tool definition; everything else stays the same.</li>`,Wl,Z,$l,E,ue='<thead><tr><th></th> <th></th></tr></thead> <tbody><tr><td><strong>Student model</strong></td> <td><a href="https://huggingface.co/Qwen/Qwen3-1.7B" rel="nofollow"><code>Qwen/Qwen3-1.7B</code></a></td></tr> <tr><td><strong>Teacher model</strong></td> <td><code>gpt-5-mini</code> via the OpenAI API</td></tr> <tr><td><strong>Environment</strong></td> <td><a href="https://github.com/huggingface/OpenEnv/tree/main/envs/reasoning_gym_env" rel="nofollow"><code>reasoning_gym_env</code></a> / <code>chain_sum</code></td></tr> <tr><td><strong>SFT trainer</strong></td> <td><a href="https://huggingface.co/docs/trl/main/en/sft_trainer" rel="nofollow">TRL <code>SFTTrainer</code></a></td></tr> <tr><td><strong>Next step</strong></td> <td><a href="end-to-end-walkthrough">End-to-end walkthrough with GRPO</a></td></tr></tbody>',Xl,Sl,Fl,G,ql,v,zl,Yl,xl,R,Hl,k,Ll,Q,Je=`You’ll also need a Hugging Face login to download the base model and push both the collected dataset
and the fine-tuned checkpoint:`,Pl,N,Ol,_,Dl,Kl,ls,V,ss,W,Te=`Use the same prompt as the <a href="end-to-end-walkthrough">GRPO tutorial</a>
so the SFT-trained model is a drop-in replacement when you continue with GRPO.`,es,$,ts,ns,as,X,Ms,S,we=`<code>DATASET_CONFIG</code> controls the difficulty of the <code>chain_sum</code> problems the environment generates:
<code>min_terms</code>/<code>max_terms</code> set how many integers are added together, and <code>min_digits</code>/<code>max_digits</code> set
how many digits each integer has. At these settings each problem is a sum of 2–3 two-digit numbers
— easy enough for <code>gpt-5-mini</code> to answer correctly ~90% of the time, which gives a clean training
signal after filtering.`,ps,F,de=`<code>N_EPISODES</code> is the number of problems to collect. 300 is enough to get ~270 correct examples after
filtering, which is sufficient for format compliance training.`,is,q,os,rs,ys,z,cs,Y,he=`<code>openenv collect</code> runs the teacher model inside the environment and records every episode — the
environment’s <code>step()</code> reward is written alongside the messages, so filtering by correctness requires
no additional scoring code.`,js,x,Us,H,be=`The command prints a live progress summary and pushes the collected episodes to the Hub as
<code>{YOUR_HF_USERNAME}/chain-sum-rollouts</code>. Pull them back to start filtering:`,ms,L,us,P,Ie=`The <code>messages</code> field stores the full conversation in standard OpenAI format (assistant messages have
a <code>tool_calls</code> list). Convert to Qwen3’s <code>&lt;tool_call&gt;</code> text format before training — GRPOTrainer
produces this same format during RL, so the SFT checkpoint becomes a direct drop-in:`,Js,O,Ts,ws,ds,D,hs,K,ge=`Keep only episodes where the teacher answered correctly. The environment’s reward signal does the
labelling — no manual annotation needed.`,bs,ll,Is,sl,fe=`<code>gpt-5-mini</code> typically scores above 90% on <code>chain_sum</code> at this difficulty, so you should end up with
~270 examples from 300 rollouts.`,gs,fs,Cs,el,As,tl,Ce=`Always look at your data before training. Automated collection can introduce unexpected patterns that the
student model will learn to imitate.`,Bs,nl,Zs,al,Ae="Things to check:",Es,Ml,Be=`<li>Does every response contain a valid <code>&lt;tool_call&gt;</code> block?</li> <li>Are the answers integers with no extra text?</li> <li>Is there any reasoning in the assistant message that you don’t want the student to learn?
(For example: an internal monologue, disclaimers, or repeated phrasing that the teacher leaked
from its own system prompt.)</li>`,Gs,vs,Rs,pl,ks,il,Ze=`Set <code>max_length</code> in <code>SFTConfig</code> to cover nearly all examples without wasting GPU memory on padding.
The 99th percentile is a good target: you truncate fewer than 1% of examples while keeping batches tight.`,Qs,ol,Ns,_s,Vs,rl,Ws,yl,Ee=`<code>assistant_only_loss=True</code> in <code>SFTConfig</code> masks the prompt tokens so the loss is computed only on the
assistant response — the <code>&lt;tool_call&gt;</code> block. This is more efficient than full-sequence training and avoids
accidentally reinforcing the system prompt wording.`,$s,cl,Xs,T,Ge=`<p>Training ~270 examples for 3 epochs takes around 5 minutes on a single A100 (40 GB). The goal is format
compliance, not task mastery — a handful of epochs is enough. Mastery comes from GRPO.</p>`,Ss,Fs,qs,jl,zs,Ul,ve=`Run both the base model and the SFT checkpoint on a held-out set and compare. The key metric for a
warm-up evaluation is <strong>format compliance</strong> — how often the model uses <code>&lt;tool_call&gt;</code> correctly — as
well as overall accuracy.`,Ys,ml,xs,ul,Re="A successful warm-up looks like this:",Hs,Jl,ke="<thead><tr><th>Metric</th> <th>Base model</th> <th>After SFT</th> <th>Delta</th></tr></thead> <tbody><tr><td>Format compliance</td> <td>~0%</td> <td>~68%</td> <td>+68 pp</td></tr> <tr><td>Accuracy</td> <td>~4%</td> <td>~68%</td> <td>+64 pp</td></tr></tbody>",Ls,Tl,Qe=`Format compliance should jump sharply from near-zero — that’s the primary goal. <code>Qwen3-1.7B</code> produces
essentially no valid <code>&lt;tool_call&gt;</code> blocks out of the box. After SFT on ~270 examples, the model reliably
uses the format, and accuracy rises in lockstep because correct format is a prerequisite for the
environment’s scorer to award any credit.`,Ps,Os,Ds,wl,Ks,dl,Ne=`The SFT checkpoint is ready to use as the starting model for GRPO. In the
<a href="end-to-end-walkthrough">end-to-end walkthrough</a>,
change one line in section 8:`,le,hl,se,bl,_e=`With format compliance already near 100%, GRPO’s <code>reward_std</code> will be non-zero from the very first
batch and the reward curve will climb immediately — no cold-start stall.`,ee,Il,Ve="<strong>Other directions:</strong>",te,gl,We=`<li><strong>Harder tasks.</strong> Increase <code>max_terms</code> or <code>max_digits</code> in <code>DATASET_CONFIG</code> and collect a new SFT set.
Once the student handles easier examples reliably, a harder GRPO phase can push further.</li> <li><strong>Different environments.</strong> The same pipeline — teacher collects → filter → SFT → GRPO — applies to
any OpenEnv environment. Swap <code>reasoning_gym_env</code> and the <code>answer</code> tool definition for your env’s
tool surface.</li> <li><strong>Larger teacher.</strong> <code>gpt-5</code> or <code>claude-opus-4</code> as teacher will yield higher-quality examples,
especially for tasks where <code>gpt-5-mini</code> struggles.</li>`,ne,fl,ae,Al,Me;return w=new He({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),d=new u({props:{title:"Collecting rollouts with OpenEnv for supervised training",local:"collecting-rollouts-with-openenv-for-supervised-training",headingTag:"h1"}}),C=new u({props:{title:"Why use an environment to collect training data",local:"why-use-an-environment-to-collect-training-data",headingTag:"h2"}}),Z=new u({props:{title:"What you’ll use",local:"what-youll-use",headingTag:"h2"}}),G=new u({props:{title:"1. Install dependencies",local:"1-install-dependencies",headingTag:"h2"}}),v=new m({props:{code:"IXBpcCUyMGluc3RhbGwlMjAtcSUyMG9wZW5haSUyMHRybCUwQSFwaXAlMjBpbnN0YWxsJTIwLXElMjBvcGVuZW52JTBBIXBpcCUyMGluc3RhbGwlMjAtcSUyMC0tbm8tZGVwcyUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZzcGFjZXMlMkZzZXJnaW9wYW5pZWdvJTJGcmVhc29uaW5nX2d5bSUwQSFwaXAlMjBpbnN0YWxsJTIwLVVxJTIwJTIydHJhbnNmb3JtZXJzJTNFJTNENS4zLjAlMjI=",highlighted:`!pip install -q openai trl
!pip install -q openenv
!pip install -q --no-deps git+https://huggingface.co/spaces/sergiopaniego/reasoning_gym
!pip install -Uq <span class="hljs-string">&quot;transformers&gt;=5.3.0&quot;</span>`,lang:"python",wrap:!1}}),R=new u({props:{title:"2. Set your credentials",local:"2-set-your-credentials",headingTag:"h2"}}),k=new m({props:{code:"aW1wb3J0JTIwZ2V0cGFzcyUyQyUyMG9zJTBBJTBBaWYlMjAlMjJPUEVOQUlfQVBJX0tFWSUyMiUyMG5vdCUyMGluJTIwb3MuZW52aXJvbiUzQSUwQSUyMCUyMCUyMCUyMG9zLmVudmlyb24lNUIlMjJPUEVOQUlfQVBJX0tFWSUyMiU1RCUyMCUzRCUyMGdldHBhc3MuZ2V0cGFzcyglMjJPcGVuQUklMjBBUEklMjBrZXklM0ElMjAlMjIp",highlighted:`<span class="hljs-keyword">import</span> getpass, os

<span class="hljs-keyword">if</span> <span class="hljs-string">&quot;OPENAI_API_KEY&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> os.environ:
    os.environ[<span class="hljs-string">&quot;OPENAI_API_KEY&quot;</span>] = getpass.getpass(<span class="hljs-string">&quot;OpenAI API key: &quot;</span>)`,lang:"python",wrap:!1}}),N=new m({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,lang:"python",wrap:!1}}),_=new m({props:{code:"WU9VUl9IRl9VU0VSTkFNRSUyMCUzRCUyMCUyMnlvdXItdXNlcm5hbWUlMjIlMjAlMjAlMjMlMjByZXBsYWNlJTIwd2l0aCUyMHlvdXIlMjBIdWdnaW5nJTIwRmFjZSUyMHVzZXJuYW1lJTBBYXNzZXJ0JTIwWU9VUl9IRl9VU0VSTkFNRSUyMCElM0QlMjAlMjJ5b3VyLXVzZXJuYW1lJTIyJTJDJTIwJTIyUmVwbGFjZSUyMFlPVVJfSEZfVVNFUk5BTUUlMjB3aXRoJTIweW91ciUyMEh1Z2dpbmclMjBGYWNlJTIwdXNlcm5hbWUlMjI=",highlighted:`YOUR_HF_USERNAME = <span class="hljs-string">&quot;your-username&quot;</span>  <span class="hljs-comment"># replace with your Hugging Face username</span>
<span class="hljs-keyword">assert</span> YOUR_HF_USERNAME != <span class="hljs-string">&quot;your-username&quot;</span>, <span class="hljs-string">&quot;Replace YOUR_HF_USERNAME with your Hugging Face username&quot;</span>`,lang:"python",wrap:!1}}),V=new u({props:{title:"3. Define the system prompt",local:"3-define-the-system-prompt",headingTag:"h2"}}),$=new m({props:{code:"U1lTVEVNX1BST01QVCUyMCUzRCUyMCUyMiUyMiUyMllvdSUyMGFyZSUyMGElMjBjYXJlZnVsJTIwYXJpdGhtZXRpYyUyMGFzc2lzdGFudC4lMEElMEFZb3UlMjB3aWxsJTIwYmUlMjBnaXZlbiUyMGElMjBjaGFpbiUyMG9mJTIwaW50ZWdlciUyMGFkZGl0aW9ucy4lMjBDb21wdXRlJTIwdGhlJTIwcmVzdWx0JTIwYW5kJTIwc3VibWl0JTIwaXQlMjBhcyUyMGElMjBzaW5nbGUlMjBudW1iZXIuJTBBJTBBUnVsZXMlM0ElMEExLiUyMFJlYWQlMjB0aGUlMjBxdWVzdGlvbiUyMGNhcmVmdWxseS4lMEEyLiUyMFVzZSUyMHRoZSUyMHRvb2wlMjAlNjBhbnN3ZXIlNjAlMjBleGFjdGx5JTIwb25jZSUyMHdpdGglMjB5b3VyJTIwZmluYWwlMjBudW1iZXIuJTBBMy4lMjBUaGUlMjBhbnN3ZXIlMjBtdXN0JTIwYmUlMjBhJTIwc2luZ2xlJTIwaW50ZWdlciUyMHdpdGglMjBubyUyMHVuaXRzJTIwb3IlMjBleHBsYW5hdGlvbi4lMEElMjIlMjIlMjI=",highlighted:`SYSTEM_PROMPT = <span class="hljs-string">&quot;&quot;&quot;You are a careful arithmetic assistant.

You will be given a chain of integer additions. Compute the result and submit it as a single number.

Rules:
1. Read the question carefully.
2. Use the tool \`answer\` exactly once with your final number.
3. The answer must be a single integer with no units or explanation.
&quot;&quot;&quot;</span>`,lang:"python",wrap:!1}}),X=new u({props:{title:"4. Configure data collection",local:"4-configure-data-collection",headingTag:"h2"}}),q=new m({props:{code:"REFUQVNFVF9DT05GSUclMjAlM0QlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjJtaW5fdGVybXMlMjIlM0ElMjAyJTJDJTBBJTIwJTIwJTIwJTIwJTIybWF4X3Rlcm1zJTIyJTNBJTIwMyUyQyUwQSUyMCUyMCUyMCUyMCUyMm1pbl9kaWdpdHMlMjIlM0ElMjAyJTJDJTBBJTIwJTIwJTIwJTIwJTIybWF4X2RpZ2l0cyUyMiUzQSUyMDIlMkMlMEElN0QlMEElMEFOX0VQSVNPREVTJTIwJTNEJTIwMzAw",highlighted:`DATASET_CONFIG = {
    <span class="hljs-string">&quot;min_terms&quot;</span>: <span class="hljs-number">2</span>,
    <span class="hljs-string">&quot;max_terms&quot;</span>: <span class="hljs-number">3</span>,
    <span class="hljs-string">&quot;min_digits&quot;</span>: <span class="hljs-number">2</span>,
    <span class="hljs-string">&quot;max_digits&quot;</span>: <span class="hljs-number">2</span>,
}

N_EPISODES = <span class="hljs-number">300</span>`,lang:"python",wrap:!1}}),z=new u({props:{title:"5. Collect rollouts with openenv collect",local:"5-collect-rollouts-with-openenv-collect",headingTag:"h2"}}),x=new m({props:{code:"aW1wb3J0JTIwanNvbiUyQyUyMHNobGV4JTBBJTBBZGF0YXNldF9jb25maWdfYXJnJTIwJTNEJTIwc2hsZXgucXVvdGUoanNvbi5kdW1wcyhEQVRBU0VUX0NPTkZJRykpJTBBc3lzdGVtX3Byb21wdF9hcmclMjAlM0QlMjBzaGxleC5xdW90ZShTWVNURU1fUFJPTVBUKSUwQWh1Yl9yZXBvX2FyZyUyMCUzRCUyMHNobGV4LnF1b3RlKGYlMjIlN0JZT1VSX0hGX1VTRVJOQU1FJTdEJTJGY2hhaW4tc3VtLXJvbGxvdXRzJTIyKSUwQSUwQSFvcGVuZW52JTIwY29sbGVjdCUyMHJlYXNvbmluZ19neW0lM0FjaGFpbl9zdW0lMjAlNUMlMEElMjAlMjAtLWJhc2UtdXJsJTIwaHR0cHMlM0ElMkYlMkZzZXJnaW9wYW5pZWdvLXJlYXNvbmluZy1neW0uaGYuc3BhY2UlMjAlNUMlMEElMjAlMjAtLXByb3ZpZGVyJTIwb3BlbmFpJTIwJTVDJTBBJTIwJTIwLS1tb2RlbCUyMGdwdC01LW1pbmklMjAlNUMlMEElMjAlMjAtLW51bS1lcGlzb2RlcyUyMCU3Qk5fRVBJU09ERVMlN0QlMjAlNUMlMEElMjAlMjAtLW1heC10b2tlbnMlMjAxMDI0JTIwJTVDJTBBJTIwJTIwLS1kYXRhc2V0LWNvbmZpZyUyMCU3QmRhdGFzZXRfY29uZmlnX2FyZyU3RCUyMCU1QyUwQSUyMCUyMC0tc3lzdGVtLXByb21wdCUyMCU3QnN5c3RlbV9wcm9tcHRfYXJnJTdEJTIwJTVDJTBBJTIwJTIwLS1wdXNoLXRvLWh1YiUyMCU3Qmh1Yl9yZXBvX2FyZyU3RCUyMCU1QyUwQSUyMCUyMC0tb3V0cHV0LWRpciUyMC4lMkZyb2xsb3V0cw==",highlighted:`<span class="hljs-keyword">import</span> json, shlex

dataset_config_arg = shlex.quote(json.dumps(DATASET_CONFIG))
system_prompt_arg = shlex.quote(SYSTEM_PROMPT)
hub_repo_arg = shlex.quote(<span class="hljs-string">f&quot;<span class="hljs-subst">{YOUR_HF_USERNAME}</span>/chain-sum-rollouts&quot;</span>)

!openenv collect reasoning_gym:chain_sum \\
  --base-url https://sergiopaniego-reasoning-gym.hf.space \\
  --provider openai \\
  --model gpt-<span class="hljs-number">5</span>-mini \\
  --num-episodes {N_EPISODES} \\
  --<span class="hljs-built_in">max</span>-tokens <span class="hljs-number">1024</span> \\
  --dataset-config {dataset_config_arg} \\
  --system-prompt {system_prompt_arg} \\
  --push-to-hub {hub_repo_arg} \\
  --output-<span class="hljs-built_in">dir</span> ./rollouts`,lang:"python",wrap:!1}}),L=new m({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoZiUyMiU3QllPVVJfSEZfVVNFUk5BTUUlN0QlMkZjaGFpbi1zdW0tcm9sbG91dHMlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQXJhd19yb2xsb3V0cyUyMCUzRCUyMGxpc3QoZHMpJTBBcHJpbnQoZiUyMkNvbGxlY3RlZCUyMCU3QmxlbihyYXdfcm9sbG91dHMpJTdEJTIwZXBpc29kZXMlMjIp",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

ds = load_dataset(<span class="hljs-string">f&quot;<span class="hljs-subst">{YOUR_HF_USERNAME}</span>/chain-sum-rollouts&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
raw_rollouts = <span class="hljs-built_in">list</span>(ds)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Collected <span class="hljs-subst">{<span class="hljs-built_in">len</span>(raw_rollouts)}</span> episodes&quot;</span>)`,lang:"python",wrap:!1}}),O=new m({props:{code:"ZGVmJTIwdG9fcXdlbjNfbWVzc2FnZXMocmVjb3JkKSUzQSUwQSUyMCUyMCUyMCUyMGNvbnZlcnRlZCUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMG1zZyUyMGluJTIwcmVjb3JkJTVCJTIybWVzc2FnZXMlMjIlNUQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMG1zZyU1QiUyMnJvbGUlMjIlNUQlMjAlM0QlM0QlMjAlMjJ0b29sJTIyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29udGludWUlMjAlMjAlMjMlMjBzdHJpcCUyMGVudmlyb25tZW50JTIwcmVzcG9uc2VzJTNCJTIwU0ZUJTIwb25seSUyMG5lZWRzJTIwdGhlJTIwYXNzaXN0YW50JTIwdHVybiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwbXNnJTVCJTIycm9sZSUyMiU1RCUyMCUzRCUzRCUyMCUyMmFzc2lzdGFudCUyMiUyMGFuZCUyMG1zZy5nZXQoJTIydG9vbF9jYWxscyUyMiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0YyUyMCUzRCUyMG1zZyU1QiUyMnRvb2xfY2FsbHMlMjIlNUQlNUIwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYXJncyUyMCUzRCUyMGpzb24ubG9hZHModGMlNUIlMjJmdW5jdGlvbiUyMiU1RCU1QiUyMmFyZ3VtZW50cyUyMiU1RCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXJfc3RyJTIwJTNEJTIwYXJncy5nZXQoJTIyYW5zd2VyJTIyJTJDJTIwJTIyJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvb2xfY2FsbF90ZXh0JTIwJTNEJTIwKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMiUzQ3Rvb2xfY2FsbCUzRSU1Q24lMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMkIlMjBqc29uLmR1bXBzKCU3QiUyMm5hbWUlMjIlM0ElMjAlMjJhbnN3ZXIlMjIlMkMlMjAlMjJhcmd1bWVudHMlMjIlM0ElMjAlN0IlMjJhbnN3ZXIlMjIlM0ElMjBhbnN3ZXJfc3RyJTdEJTdEKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyQiUyMCUyMiU1Q24lM0MlMkZ0b29sX2NhbGwlM0UlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29udmVydGVkLmFwcGVuZCglN0IlMjJyb2xlJTIyJTNBJTIwJTIyYXNzaXN0YW50JTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMHRvb2xfY2FsbF90ZXh0JTdEKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb252ZXJ0ZWQuYXBwZW5kKG1zZyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlMjJtZXNzYWdlcyUyMiUzQSUyMGNvbnZlcnRlZCUyQyUyMCUyMnJld2FyZCUyMiUzQSUyMHJlY29yZCU1QiUyMnJld2FyZCUyMiU1RCU3RCUwQSUwQXJvbGxvdXRzJTIwJTNEJTIwJTVCdG9fcXdlbjNfbWVzc2FnZXMociklMjBmb3IlMjByJTIwaW4lMjByYXdfcm9sbG91dHMlNUQ=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">to_qwen3_messages</span>(<span class="hljs-params">record</span>):
    converted = []
    <span class="hljs-keyword">for</span> msg <span class="hljs-keyword">in</span> record[<span class="hljs-string">&quot;messages&quot;</span>]:
        <span class="hljs-keyword">if</span> msg[<span class="hljs-string">&quot;role&quot;</span>] == <span class="hljs-string">&quot;tool&quot;</span>:
            <span class="hljs-keyword">continue</span>  <span class="hljs-comment"># strip environment responses; SFT only needs the assistant turn</span>
        <span class="hljs-keyword">if</span> msg[<span class="hljs-string">&quot;role&quot;</span>] == <span class="hljs-string">&quot;assistant&quot;</span> <span class="hljs-keyword">and</span> msg.get(<span class="hljs-string">&quot;tool_calls&quot;</span>):
            tc = msg[<span class="hljs-string">&quot;tool_calls&quot;</span>][<span class="hljs-number">0</span>]
            args = json.loads(tc[<span class="hljs-string">&quot;function&quot;</span>][<span class="hljs-string">&quot;arguments&quot;</span>])
            answer_str = args.get(<span class="hljs-string">&quot;answer&quot;</span>, <span class="hljs-string">&quot;&quot;</span>)
            tool_call_text = (
                <span class="hljs-string">&quot;&lt;tool_call&gt;\\n&quot;</span>
                + json.dumps({<span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;answer&quot;</span>, <span class="hljs-string">&quot;arguments&quot;</span>: {<span class="hljs-string">&quot;answer&quot;</span>: answer_str}})
                + <span class="hljs-string">&quot;\\n&lt;/tool_call&gt;&quot;</span>
            )
            converted.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: tool_call_text})
        <span class="hljs-keyword">else</span>:
            converted.append(msg)
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;messages&quot;</span>: converted, <span class="hljs-string">&quot;reward&quot;</span>: record[<span class="hljs-string">&quot;reward&quot;</span>]}

rollouts = [to_qwen3_messages(r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> raw_rollouts]`,lang:"python",wrap:!1}}),D=new u({props:{title:"6. Filter the dataset",local:"6-filter-the-dataset",headingTag:"h2"}}),ll=new m({props:{code:"Y29ycmVjdCUyMCUzRCUyMCU1QnIlMjBmb3IlMjByJTIwaW4lMjByb2xsb3V0cyUyMGlmJTIwciU1QiUyMnJld2FyZCUyMiU1RCUyMCUzRCUzRCUyMDEuMCU1RCUwQXByaW50KGYlMjJDb3JyZWN0JTNBJTIwJTdCbGVuKGNvcnJlY3QpJTdEJTIwJTJGJTIwJTdCbGVuKHJvbGxvdXRzKSU3RCUyMCglN0JsZW4oY29ycmVjdCklMkZsZW4ocm9sbG91dHMpJTNBLjElMjUlN0QpJTIyKQ==",highlighted:`correct = [r <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> rollouts <span class="hljs-keyword">if</span> r[<span class="hljs-string">&quot;reward&quot;</span>] == <span class="hljs-number">1.0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Correct: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(correct)}</span> / <span class="hljs-subst">{<span class="hljs-built_in">len</span>(rollouts)}</span> (<span class="hljs-subst">{<span class="hljs-built_in">len</span>(correct)/<span class="hljs-built_in">len</span>(rollouts):<span class="hljs-number">.1</span>%}</span>)&quot;</span>)`,lang:"python",wrap:!1}}),el=new u({props:{title:"7. Inspect the dataset before training",local:"7-inspect-the-dataset-before-training",headingTag:"h2"}}),nl=new m({props:{code:"aW1wb3J0JTIwcmFuZG9tJTBBJTBBZm9yJTIwcm93JTIwaW4lMjByYW5kb20uc2FtcGxlKGNvcnJlY3QlMkMlMjAzKSUzQSUwQSUyMCUyMCUyMCUyMHF1ZXN0aW9uJTIwJTNEJTIwcm93JTVCJTIybWVzc2FnZXMlMjIlNUQlNUIwJTVEJTVCJTIyY29udGVudCUyMiU1RCUwQSUyMCUyMCUyMCUyMHJlc3BvbnNlJTIwJTNEJTIwcm93JTVCJTIybWVzc2FnZXMlMjIlNUQlNUIxJTVEJTVCJTIyY29udGVudCUyMiU1RCUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJRJTNBJTIwJTdCcXVlc3Rpb24lN0QlMjIpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMkElM0ElMjAlN0JyZXNwb25zZSU3RCUyMiklMEElMjAlMjAlMjAlMjBwcmludCgp",highlighted:`<span class="hljs-keyword">import</span> random

<span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> random.sample(correct, <span class="hljs-number">3</span>):
    question = row[<span class="hljs-string">&quot;messages&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>]
    response = row[<span class="hljs-string">&quot;messages&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;content&quot;</span>]
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Q: <span class="hljs-subst">{question}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;A: <span class="hljs-subst">{response}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>()`,lang:"python",wrap:!1}}),pl=new u({props:{title:"8. Measure token lengths",local:"8-measure-token-lengths",headingTag:"h2"}}),ol=new m({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJRd2VuJTJGUXdlbjMtMS43QiUyMiklMEElMEFsZW5ndGhzJTIwJTNEJTIwJTVCJTVEJTBBZm9yJTIwcm93JTIwaW4lMjBjb3JyZWN0JTNBJTBBJTIwJTIwJTIwJTIwdGV4dCUyMCUzRCUyMHRva2VuaXplci5hcHBseV9jaGF0X3RlbXBsYXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJvdyU1QiUyMm1lc3NhZ2VzJTIyJTVEJTJDJTIwdG9rZW5pemUlM0RGYWxzZSUyQyUyMGFkZF9nZW5lcmF0aW9uX3Byb21wdCUzREZhbHNlJTBBJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMGlkcyUyMCUzRCUyMHRva2VuaXplci5lbmNvZGUodGV4dCklMEElMjAlMjAlMjAlMjBsZW5ndGhzLmFwcGVuZChsZW4oaWRzKSklMEElMEFsZW5ndGhzJTIwJTNEJTIwbnAuYXJyYXkobGVuZ3RocyklMEFNQVhfU0VRX0xFTiUyMCUzRCUyMGludChucC5wZXJjZW50aWxlKGxlbmd0aHMlMkMlMjA5OSkpJTIwJTJCJTIwMTYlMEElMEFwcmludCglMEElMjAlMjAlMjAlMjBmJTIycDUwJTNEJTdCbnAucGVyY2VudGlsZShsZW5ndGhzJTJDJTIwNTApJTNBLjBmJTdEJTIwJTIwJTIyJTBBJTIwJTIwJTIwJTIwZiUyMnA5NSUzRCU3Qm5wLnBlcmNlbnRpbGUobGVuZ3RocyUyQyUyMDk1KSUzQS4wZiU3RCUyMCUyMCUyMiUwQSUyMCUyMCUyMCUyMGYlMjJwOTklM0QlN0JucC5wZXJjZW50aWxlKGxlbmd0aHMlMkMlMjA5OSklM0EuMGYlN0QlMjAlMjAlMjIlMEElMjAlMjAlMjAlMjBmJTIybWF4JTNEJTdCbGVuZ3Rocy5tYXgoKSU3RCUyMiUwQSklMEFwcmludChmJTIyU2V0dGluZyUyME1BWF9TRVFfTEVOJTIwJTNEJTIwJTdCTUFYX1NFUV9MRU4lN0QlMjIp",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;Qwen/Qwen3-1.7B&quot;</span>)

lengths = []
<span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> correct:
    text = tokenizer.apply_chat_template(
        row[<span class="hljs-string">&quot;messages&quot;</span>], tokenize=<span class="hljs-literal">False</span>, add_generation_prompt=<span class="hljs-literal">False</span>
    )
    ids = tokenizer.encode(text)
    lengths.append(<span class="hljs-built_in">len</span>(ids))

lengths = np.array(lengths)
MAX_SEQ_LEN = <span class="hljs-built_in">int</span>(np.percentile(lengths, <span class="hljs-number">99</span>)) + <span class="hljs-number">16</span>

<span class="hljs-built_in">print</span>(
    <span class="hljs-string">f&quot;p50=<span class="hljs-subst">{np.percentile(lengths, <span class="hljs-number">50</span>):<span class="hljs-number">.0</span>f}</span>  &quot;</span>
    <span class="hljs-string">f&quot;p95=<span class="hljs-subst">{np.percentile(lengths, <span class="hljs-number">95</span>):<span class="hljs-number">.0</span>f}</span>  &quot;</span>
    <span class="hljs-string">f&quot;p99=<span class="hljs-subst">{np.percentile(lengths, <span class="hljs-number">99</span>):<span class="hljs-number">.0</span>f}</span>  &quot;</span>
    <span class="hljs-string">f&quot;max=<span class="hljs-subst">{lengths.<span class="hljs-built_in">max</span>()}</span>&quot;</span>
)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Setting MAX_SEQ_LEN = <span class="hljs-subst">{MAX_SEQ_LEN}</span>&quot;</span>)`,lang:"python",wrap:!1}}),rl=new u({props:{title:"9. Fine-tune with SFTTrainer",local:"9-fine-tune-with-sfttrainer",headingTag:"h2"}}),cl=new m({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQWZyb20lMjB0cmwlMjBpbXBvcnQlMjBTRlRDb25maWclMkMlMjBTRlRUcmFpbmVyJTBBJTBBZGF0YXNldCUyMCUzRCUyMERhdGFzZXQuZnJvbV9saXN0KCU1QiU3QiUyMm1lc3NhZ2VzJTIyJTNBJTIwciU1QiUyMm1lc3NhZ2VzJTIyJTVEJTdEJTIwZm9yJTIwciUyMGluJTIwY29ycmVjdCU1RCklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJRd2VuJTJGUXdlbjMtMS43QiUyMiklMEElMEFzZnRfY29uZmlnJTIwJTNEJTIwU0ZUQ29uZmlnKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJyZWFzb25pbmctZ3ltLWNoYWluLXN1bS1Rd2VuMy0xLjdCLXNmdCUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9sZW5ndGglM0RNQVhfU0VRX0xFTiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QzJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNSUyQyUwQSUyMCUyMCUyMCUyMHdhcm11cF9zdGVwcyUzRDEwJTJDJTBBJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyX3R5cGUlM0QlMjJjb3NpbmUlMjIlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNENSUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfc3RyYXRlZ3klM0QlMjJubyUyMiUyQyUwQSUyMCUyMCUyMCUyMGFzc2lzdGFudF9vbmx5X2xvc3MlM0RUcnVlJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBTRlRUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHByb2Nlc3NpbmdfY2xhc3MlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEc2Z0X2NvbmZpZyUyQyUwQSklMEElMEF0cmFpbmVyLnRyYWluKCklMEF0cmFpbmVyLnB1c2hfdG9faHViKGNvbW1pdF9tZXNzYWdlJTNEJTIyU0ZUJTIwd2FybS11cCUyMG9uJTIwcmVhc29uaW5nX2d5bSUyMGNoYWluX3N1bSUyMik=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM
<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTConfig, SFTTrainer

dataset = Dataset.from_list([{<span class="hljs-string">&quot;messages&quot;</span>: r[<span class="hljs-string">&quot;messages&quot;</span>]} <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> correct])

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;Qwen/Qwen3-1.7B&quot;</span>)

sft_config = SFTConfig(
    output_dir=<span class="hljs-string">&quot;reasoning-gym-chain-sum-Qwen3-1.7B-sft&quot;</span>,
    max_length=MAX_SEQ_LEN,
    num_train_epochs=<span class="hljs-number">3</span>,
    per_device_train_batch_size=<span class="hljs-number">4</span>,
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    warmup_steps=<span class="hljs-number">10</span>,
    lr_scheduler_type=<span class="hljs-string">&quot;cosine&quot;</span>,
    logging_steps=<span class="hljs-number">5</span>,
    save_strategy=<span class="hljs-string">&quot;no&quot;</span>,
    assistant_only_loss=<span class="hljs-literal">True</span>,
)

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    processing_class=tokenizer,
    args=sft_config,
)

trainer.train()
trainer.push_to_hub(commit_message=<span class="hljs-string">&quot;SFT warm-up on reasoning_gym chain_sum&quot;</span>)`,lang:"python",wrap:!1}}),jl=new u({props:{title:"10. Evaluate: before vs after",local:"10-evaluate-before-vs-after",headingTag:"h2"}}),ml=new m({props:{code:"aW1wb3J0JTIwcmUlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwcGlwZWxpbmUlMEFmcm9tJTIwcmVhc29uaW5nX2d5bV9lbnYuY2xpZW50JTIwaW1wb3J0JTIwUmVhc29uaW5nR3ltRW52JTBBZnJvbSUyMHJlYXNvbmluZ19neW1fZW52Lm1vZGVscyUyMGltcG9ydCUyMFJlYXNvbmluZ0d5bUFjdGlvbiUwQSUwQSUwQWFzeW5jJTIwZGVmJTIwZXZhbHVhdGVfbW9kZWwobW9kZWxfbmFtZSUyQyUyMG5fZXZhbCUzRDUwJTJDJTIwc2VlZCUzRDk5OSklM0ElMEElMjAlMjAlMjAlMjBnZW4lMjAlM0QlMjBwaXBlbGluZSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0ZXh0LWdlbmVyYXRpb24lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsX25hbWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0Rtb2RlbF9uYW1lJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkdHlwZSUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwZ2VuLm1vZGVsLmdlbmVyYXRpb25fY29uZmlnLm1heF9sZW5ndGglMjAlM0QlMjBOb25lJTBBJTIwJTIwJTIwJTIwdG9rJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSklMEElMjAlMjAlMjAlMjBlbnYlMjAlM0QlMjBSZWFzb25pbmdHeW1FbnYoYmFzZV91cmwlM0QlMjJodHRwcyUzQSUyRiUyRnNlcmdpb3BhbmllZ28tcmVhc29uaW5nLWd5bS5oZi5zcGFjZSUyMiklMEElMEElMjAlMjAlMjAlMjBvYnMlMjAlM0QlMjBhd2FpdCUyMGVudi5yZXNldCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkYXRhc2V0X25hbWUlM0QlMjJjaGFpbl9zdW0lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkYXRhc2V0X2NvbmZpZyUzRERBVEFTRVRfQ09ORklHJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VlZCUzRHNlZWQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzaXplJTNEbl9ldmFsJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMHJld2FyZHMlMkMlMjBmb3JtYXRfaGl0cyUyMCUzRCUyMCU1QiU1RCUyQyUyMDAlMEElMEElMjAlMjAlMjAlMjBmb3IlMjBpJTIwaW4lMjByYW5nZShuX2V2YWwpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBpJTIwJTNFJTIwMCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9icyUyMCUzRCUyMGF3YWl0JTIwZW52LnJlc2V0KCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBxdWVzdGlvbiUyMCUzRCUyMG9icy5vYnNlcnZhdGlvbi5xdWVzdGlvbiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1lc3NhZ2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjBTWVNURU1fUFJPTVBUJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwcXVlc3Rpb24lN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcm9tcHQlMjAlM0QlMjB0b2suYXBwbHlfY2hhdF90ZW1wbGF0ZSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtZXNzYWdlcyUyQyUyMHRva2VuaXplJTNERmFsc2UlMkMlMjBhZGRfZ2VuZXJhdGlvbl9wcm9tcHQlM0RUcnVlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbXBsZXRpb24lMjAlM0QlMjBnZW4ocHJvbXB0JTJDJTIwbWF4X25ld190b2tlbnMlM0Q2NCklNUIwJTVEJTVCJTIyZ2VuZXJhdGVkX3RleHQlMjIlNUQlNUJsZW4ocHJvbXB0KSUzQSU1RCUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG0lMjAlM0QlMjByZS5zZWFyY2gociclMjJhbnN3ZXIlMjIlNUNzKiUzQSU1Q3MqJTIyJTNGKCU1Q2QlMkIpJTIyJTNGJyUyQyUyMGNvbXBsZXRpb24pJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBtJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9ybWF0X2hpdHMlMjAlMkIlM0QlMjAxJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYW5zd2VyJTIwJTNEJTIwbS5ncm91cCgxKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBudW1zJTIwJTNEJTIwcmUuZmluZGFsbChyJTIyJTVDYiglNUNkJTJCKSU1Q2IlMjIlMkMlMjBjb21wbGV0aW9uKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFuc3dlciUyMCUzRCUyMG51bXMlNUItMSU1RCUyMGlmJTIwbnVtcyUyMGVsc2UlMjAlMjIwJTIyJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwYXdhaXQlMjBlbnYuc3RlcChSZWFzb25pbmdHeW1BY3Rpb24oYW5zd2VyJTNEYW5zd2VyKSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZChmbG9hdChyZXN1bHQub2JzZXJ2YXRpb24uc2NvcmUlMjBvciUyMDAuMCkpJTBBJTBBJTIwJTIwJTIwJTIwYXdhaXQlMjBlbnYuY2xvc2UoKSUwQSUyMCUyMCUyMCUyMGRlbCUyMGdlbiUyMCUyMCUyMyUyMGZyZWUlMjBHUFUlMjBtZW1vcnklMjBiZWZvcmUlMjBsb2FkaW5nJTIwdGhlJTIwbmV4dCUyMG1vZGVsJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWNjdXJhY3klMjIlM0ElMjBzdW0ocmV3YXJkcyklMjAlMkYlMjBsZW4ocmV3YXJkcyklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJmb3JtYXRfY29tcGxpYW5jZSUyMiUzQSUyMGZvcm1hdF9oaXRzJTIwJTJGJTIwbl9ldmFsJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTBBJTBBJTBBYmFzZV9tZXRyaWNzJTIwJTNEJTIwYXdhaXQlMjBldmFsdWF0ZV9tb2RlbCglMjJRd2VuJTJGUXdlbjMtMS43QiUyMiklMEFzZnRfbWV0cmljcyUyMCUzRCUyMGF3YWl0JTIwZXZhbHVhdGVfbW9kZWwoZiUyMiU3QllPVVJfSEZfVVNFUk5BTUUlN0QlMkZyZWFzb25pbmctZ3ltLWNoYWluLXN1bS1Rd2VuMy0xLjdCLXNmdCUyMiklMEElMEFwcmludChmJTIyJTVDbiU3QidNZXRyaWMnJTNBJTNDMjUlN0QlMjAlN0InQmFzZSUyMG1vZGVsJyUzQSUzRTEyJTdEJTIwJTdCJ0FmdGVyJTIwU0ZUJyUzQSUzRTEyJTdEJTIwJTdCJ0RlbHRhJyUzQSUzRTEwJTdEJTIyKSUwQXByaW50KCUyMi0lMjIlMjAqJTIwNjIpJTBBZm9yJTIwa2V5JTJDJTIwbGFiZWwlMjBpbiUyMCU1QiglMjJmb3JtYXRfY29tcGxpYW5jZSUyMiUyQyUyMCUyMkZvcm1hdCUyMGNvbXBsaWFuY2UlMjIpJTJDJTIwKCUyMmFjY3VyYWN5JTIyJTJDJTIwJTIyQWNjdXJhY3klMjIpJTVEJTNBJTBBJTIwJTIwJTIwJTIwYiUyQyUyMHMlMjAlM0QlMjBiYXNlX21ldHJpY3MlNUJrZXklNUQlMkMlMjBzZnRfbWV0cmljcyU1QmtleSU1RCUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlN0JsYWJlbCUzQSUzQzI1JTdEJTIwJTdCYiUzQSUzRTEyLjElMjUlN0QlMjAlN0JzJTNBJTNFMTIuMSUyNSU3RCUyMCU3QihzJTIwLSUyMGIpJTIwKiUyMDEwMCUzQSUzRSUyQjkuMWYlN0QlMjBwcCUyMik=",highlighted:`<span class="hljs-keyword">import</span> re
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">from</span> reasoning_gym_env.client <span class="hljs-keyword">import</span> ReasoningGymEnv
<span class="hljs-keyword">from</span> reasoning_gym_env.models <span class="hljs-keyword">import</span> ReasoningGymAction


<span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">evaluate_model</span>(<span class="hljs-params">model_name, n_eval=<span class="hljs-number">50</span>, seed=<span class="hljs-number">999</span></span>):
    gen = pipeline(
        <span class="hljs-string">&quot;text-generation&quot;</span>,
        model=model_name,
        tokenizer=model_name,
        device_map=<span class="hljs-string">&quot;auto&quot;</span>,
        dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    )
    gen.model.generation_config.max_length = <span class="hljs-literal">None</span>
    tok = AutoTokenizer.from_pretrained(model_name)
    env = ReasoningGymEnv(base_url=<span class="hljs-string">&quot;https://sergiopaniego-reasoning-gym.hf.space&quot;</span>)

    obs = <span class="hljs-keyword">await</span> env.reset(
        dataset_name=<span class="hljs-string">&quot;chain_sum&quot;</span>,
        dataset_config=DATASET_CONFIG,
        seed=seed,
        size=n_eval,
    )

    rewards, format_hits = [], <span class="hljs-number">0</span>

    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_eval):
        <span class="hljs-keyword">if</span> i &gt; <span class="hljs-number">0</span>:
            obs = <span class="hljs-keyword">await</span> env.reset()

        question = obs.observation.question
        messages = [
            {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: SYSTEM_PROMPT},
            {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: question},
        ]
        prompt = tok.apply_chat_template(
            messages, tokenize=<span class="hljs-literal">False</span>, add_generation_prompt=<span class="hljs-literal">True</span>
        )
        completion = gen(prompt, max_new_tokens=<span class="hljs-number">64</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>][<span class="hljs-built_in">len</span>(prompt):]

        m = re.search(<span class="hljs-string">r&#x27;&quot;answer&quot;\\s*:\\s*&quot;?(\\d+)&quot;?&#x27;</span>, completion)
        <span class="hljs-keyword">if</span> m:
            format_hits += <span class="hljs-number">1</span>
            answer = m.group(<span class="hljs-number">1</span>)
        <span class="hljs-keyword">else</span>:
            nums = re.findall(<span class="hljs-string">r&quot;\\b(\\d+)\\b&quot;</span>, completion)
            answer = nums[-<span class="hljs-number">1</span>] <span class="hljs-keyword">if</span> nums <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;0&quot;</span>

        result = <span class="hljs-keyword">await</span> env.step(ReasoningGymAction(answer=answer))
        rewards.append(<span class="hljs-built_in">float</span>(result.observation.score <span class="hljs-keyword">or</span> <span class="hljs-number">0.0</span>))

    <span class="hljs-keyword">await</span> env.close()
    <span class="hljs-keyword">del</span> gen  <span class="hljs-comment"># free GPU memory before loading the next model</span>

    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;accuracy&quot;</span>: <span class="hljs-built_in">sum</span>(rewards) / <span class="hljs-built_in">len</span>(rewards),
        <span class="hljs-string">&quot;format_compliance&quot;</span>: format_hits / n_eval,
    }


base_metrics = <span class="hljs-keyword">await</span> evaluate_model(<span class="hljs-string">&quot;Qwen/Qwen3-1.7B&quot;</span>)
sft_metrics = <span class="hljs-keyword">await</span> evaluate_model(<span class="hljs-string">f&quot;<span class="hljs-subst">{YOUR_HF_USERNAME}</span>/reasoning-gym-chain-sum-Qwen3-1.7B-sft&quot;</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n<span class="hljs-subst">{<span class="hljs-string">&#x27;Metric&#x27;</span>:&lt;<span class="hljs-number">25</span>}</span> <span class="hljs-subst">{<span class="hljs-string">&#x27;Base model&#x27;</span>:&gt;<span class="hljs-number">12</span>}</span> <span class="hljs-subst">{<span class="hljs-string">&#x27;After SFT&#x27;</span>:&gt;<span class="hljs-number">12</span>}</span> <span class="hljs-subst">{<span class="hljs-string">&#x27;Delta&#x27;</span>:&gt;<span class="hljs-number">10</span>}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;-&quot;</span> * <span class="hljs-number">62</span>)
<span class="hljs-keyword">for</span> key, label <span class="hljs-keyword">in</span> [(<span class="hljs-string">&quot;format_compliance&quot;</span>, <span class="hljs-string">&quot;Format compliance&quot;</span>), (<span class="hljs-string">&quot;accuracy&quot;</span>, <span class="hljs-string">&quot;Accuracy&quot;</span>)]:
    b, s = base_metrics[key], sft_metrics[key]
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{label:&lt;<span class="hljs-number">25</span>}</span> <span class="hljs-subst">{b:&gt;<span class="hljs-number">12.1</span>%}</span> <span class="hljs-subst">{s:&gt;<span class="hljs-number">12.1</span>%}</span> <span class="hljs-subst">{(s - b) * <span class="hljs-number">100</span>:&gt;+<span class="hljs-number">9.1</span>f}</span> pp&quot;</span>)`,lang:"python",wrap:!1}}),wl=new u({props:{title:"11. Where to go next: GRPO",local:"11-where-to-go-next-grpo",headingTag:"h2"}}),hl=new m({props:{code:"JTIzJTIwQmVmb3JlJTIwKGNvbGQtc3RhcnQlMjBmcm9tJTIwdGhlJTIwYmFzZSUyMG1vZGVsKSUzQSUwQU1PREVMX05BTUUlMjAlM0QlMjAlMjJRd2VuJTJGUXdlbjMtMS43QiUyMiUwQSUwQSUyMyUyMEFmdGVyJTIwKHdhcm0tc3RhcnQlMjBmcm9tJTIweW91ciUyMFNGVCUyMGNoZWNrcG9pbnQpJTNBJTBBTU9ERUxfTkFNRSUyMCUzRCUyMGYlMjIlN0JZT1VSX0hGX1VTRVJOQU1FJTdEJTJGcmVhc29uaW5nLWd5bS1jaGFpbi1zdW0tUXdlbjMtMS43Qi1zZnQlMjI=",highlighted:`<span class="hljs-comment"># Before (cold-start from the base model):</span>
MODEL_NAME = <span class="hljs-string">&quot;Qwen/Qwen3-1.7B&quot;</span>

<span class="hljs-comment"># After (warm-start from your SFT checkpoint):</span>
MODEL_NAME = <span class="hljs-string">f&quot;<span class="hljs-subst">{YOUR_HF_USERNAME}</span>/reasoning-gym-chain-sum-Qwen3-1.7B-sft&quot;</span>`,lang:"python",wrap:!1}}),fl=new Le({props:{source:"https://github.com/huggingface/openenv/blob/main/docs/source/tutorials/sft-warmup.md"}}),{c(){J=M("meta"),Bl=n(),Cl=M("p"),Zl=n(),i(w.$$.fragment),El=n(),i(d.$$.fragment),Gl=n(),h=M("p"),h.innerHTML=oe,vl=n(),b=M("p"),b.innerHTML=re,Rl=n(),I=M("p"),I.textContent=ye,kl=n(),g=M("ol"),g.innerHTML=ce,Ql=n(),f=M("p"),f.innerHTML=je,Nl=n(),i(C.$$.fragment),_l=n(),A=M("p"),A.textContent=Ue,Vl=n(),B=M("ul"),B.innerHTML=me,Wl=n(),i(Z.$$.fragment),$l=n(),E=M("table"),E.innerHTML=ue,Xl=n(),Sl=M("hr"),Fl=n(),i(G.$$.fragment),ql=n(),i(v.$$.fragment),zl=n(),Yl=M("hr"),xl=n(),i(R.$$.fragment),Hl=n(),i(k.$$.fragment),Ll=n(),Q=M("p"),Q.textContent=Je,Pl=n(),i(N.$$.fragment),Ol=n(),i(_.$$.fragment),Dl=n(),Kl=M("hr"),ls=n(),i(V.$$.fragment),ss=n(),W=M("p"),W.innerHTML=Te,es=n(),i($.$$.fragment),ts=n(),ns=M("hr"),as=n(),i(X.$$.fragment),Ms=n(),S=M("p"),S.innerHTML=we,ps=n(),F=M("p"),F.innerHTML=de,is=n(),i(q.$$.fragment),os=n(),rs=M("hr"),ys=n(),i(z.$$.fragment),cs=n(),Y=M("p"),Y.innerHTML=he,js=n(),i(x.$$.fragment),Us=n(),H=M("p"),H.innerHTML=be,ms=n(),i(L.$$.fragment),us=n(),P=M("p"),P.innerHTML=Ie,Js=n(),i(O.$$.fragment),Ts=n(),ws=M("hr"),ds=n(),i(D.$$.fragment),hs=n(),K=M("p"),K.textContent=ge,bs=n(),i(ll.$$.fragment),Is=n(),sl=M("p"),sl.innerHTML=fe,gs=n(),fs=M("hr"),Cs=n(),i(el.$$.fragment),As=n(),tl=M("p"),tl.textContent=Ce,Bs=n(),i(nl.$$.fragment),Zs=n(),al=M("p"),al.textContent=Ae,Es=n(),Ml=M("ul"),Ml.innerHTML=Be,Gs=n(),vs=M("hr"),Rs=n(),i(pl.$$.fragment),ks=n(),il=M("p"),il.innerHTML=Ze,Qs=n(),i(ol.$$.fragment),Ns=n(),_s=M("hr"),Vs=n(),i(rl.$$.fragment),Ws=n(),yl=M("p"),yl.innerHTML=Ee,$s=n(),i(cl.$$.fragment),Xs=n(),T=M("blockquote"),T.innerHTML=Ge,Ss=n(),Fs=M("hr"),qs=n(),i(jl.$$.fragment),zs=n(),Ul=M("p"),Ul.innerHTML=ve,Ys=n(),i(ml.$$.fragment),xs=n(),ul=M("p"),ul.textContent=Re,Hs=n(),Jl=M("table"),Jl.innerHTML=ke,Ls=n(),Tl=M("p"),Tl.innerHTML=Qe,Ps=n(),Os=M("hr"),Ds=n(),i(wl.$$.fragment),Ks=n(),dl=M("p"),dl.innerHTML=Ne,le=n(),i(hl.$$.fragment),se=n(),bl=M("p"),bl.innerHTML=_e,ee=n(),Il=M("p"),Il.innerHTML=Ve,te=n(),gl=M("ul"),gl.innerHTML=We,ne=n(),i(fl.$$.fragment),ae=n(),Al=M("p"),this.h()},l(l){const s=Ye("svelte-u9bgzb",document.head);J=p(s,"META",{name:!0,content:!0}),s.forEach(e),Bl=a(l),Cl=p(l,"P",{}),$e(Cl).forEach(e),Zl=a(l),o(w.$$.fragment,l),El=a(l),o(d.$$.fragment,l),Gl=a(l),h=p(l,"P",{"data-svelte-h":!0}),r(h)!=="svelte-1mb1q37"&&(h.innerHTML=oe),vl=a(l),b=p(l,"P",{"data-svelte-h":!0}),r(b)!=="svelte-n1a38f"&&(b.innerHTML=re),Rl=a(l),I=p(l,"P",{"data-svelte-h":!0}),r(I)!=="svelte-cj2ozg"&&(I.textContent=ye),kl=a(l),g=p(l,"OL",{"data-svelte-h":!0}),r(g)!=="svelte-1mxsm9c"&&(g.innerHTML=ce),Ql=a(l),f=p(l,"P",{"data-svelte-h":!0}),r(f)!=="svelte-1rn2nlx"&&(f.innerHTML=je),Nl=a(l),o(C.$$.fragment,l),_l=a(l),A=p(l,"P",{"data-svelte-h":!0}),r(A)!=="svelte-2qxvgr"&&(A.textContent=Ue),Vl=a(l),B=p(l,"UL",{"data-svelte-h":!0}),r(B)!=="svelte-16gl30a"&&(B.innerHTML=me),Wl=a(l),o(Z.$$.fragment,l),$l=a(l),E=p(l,"TABLE",{"data-svelte-h":!0}),r(E)!=="svelte-148mvz9"&&(E.innerHTML=ue),Xl=a(l),Sl=p(l,"HR",{}),Fl=a(l),o(G.$$.fragment,l),ql=a(l),o(v.$$.fragment,l),zl=a(l),Yl=p(l,"HR",{}),xl=a(l),o(R.$$.fragment,l),Hl=a(l),o(k.$$.fragment,l),Ll=a(l),Q=p(l,"P",{"data-svelte-h":!0}),r(Q)!=="svelte-1t2mhms"&&(Q.textContent=Je),Pl=a(l),o(N.$$.fragment,l),Ol=a(l),o(_.$$.fragment,l),Dl=a(l),Kl=p(l,"HR",{}),ls=a(l),o(V.$$.fragment,l),ss=a(l),W=p(l,"P",{"data-svelte-h":!0}),r(W)!=="svelte-18uzm6p"&&(W.innerHTML=Te),es=a(l),o($.$$.fragment,l),ts=a(l),ns=p(l,"HR",{}),as=a(l),o(X.$$.fragment,l),Ms=a(l),S=p(l,"P",{"data-svelte-h":!0}),r(S)!=="svelte-poatas"&&(S.innerHTML=we),ps=a(l),F=p(l,"P",{"data-svelte-h":!0}),r(F)!=="svelte-158jha2"&&(F.innerHTML=de),is=a(l),o(q.$$.fragment,l),os=a(l),rs=p(l,"HR",{}),ys=a(l),o(z.$$.fragment,l),cs=a(l),Y=p(l,"P",{"data-svelte-h":!0}),r(Y)!=="svelte-1w7kvuu"&&(Y.innerHTML=he),js=a(l),o(x.$$.fragment,l),Us=a(l),H=p(l,"P",{"data-svelte-h":!0}),r(H)!=="svelte-ptdf38"&&(H.innerHTML=be),ms=a(l),o(L.$$.fragment,l),us=a(l),P=p(l,"P",{"data-svelte-h":!0}),r(P)!=="svelte-1pn841m"&&(P.innerHTML=Ie),Js=a(l),o(O.$$.fragment,l),Ts=a(l),ws=p(l,"HR",{}),ds=a(l),o(D.$$.fragment,l),hs=a(l),K=p(l,"P",{"data-svelte-h":!0}),r(K)!=="svelte-1vi36q9"&&(K.textContent=ge),bs=a(l),o(ll.$$.fragment,l),Is=a(l),sl=p(l,"P",{"data-svelte-h":!0}),r(sl)!=="svelte-1wrgc3t"&&(sl.innerHTML=fe),gs=a(l),fs=p(l,"HR",{}),Cs=a(l),o(el.$$.fragment,l),As=a(l),tl=p(l,"P",{"data-svelte-h":!0}),r(tl)!=="svelte-6lfca0"&&(tl.textContent=Ce),Bs=a(l),o(nl.$$.fragment,l),Zs=a(l),al=p(l,"P",{"data-svelte-h":!0}),r(al)!=="svelte-1jmsvh4"&&(al.textContent=Ae),Es=a(l),Ml=p(l,"UL",{"data-svelte-h":!0}),r(Ml)!=="svelte-vbk6t4"&&(Ml.innerHTML=Be),Gs=a(l),vs=p(l,"HR",{}),Rs=a(l),o(pl.$$.fragment,l),ks=a(l),il=p(l,"P",{"data-svelte-h":!0}),r(il)!=="svelte-d6vup6"&&(il.innerHTML=Ze),Qs=a(l),o(ol.$$.fragment,l),Ns=a(l),_s=p(l,"HR",{}),Vs=a(l),o(rl.$$.fragment,l),Ws=a(l),yl=p(l,"P",{"data-svelte-h":!0}),r(yl)!=="svelte-1uo0pb3"&&(yl.innerHTML=Ee),$s=a(l),o(cl.$$.fragment,l),Xs=a(l),T=p(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(T)!=="svelte-l32ipe"&&(T.innerHTML=Ge),Ss=a(l),Fs=p(l,"HR",{}),qs=a(l),o(jl.$$.fragment,l),zs=a(l),Ul=p(l,"P",{"data-svelte-h":!0}),r(Ul)!=="svelte-1bax8nc"&&(Ul.innerHTML=ve),Ys=a(l),o(ml.$$.fragment,l),xs=a(l),ul=p(l,"P",{"data-svelte-h":!0}),r(ul)!=="svelte-1udb8dr"&&(ul.textContent=Re),Hs=a(l),Jl=p(l,"TABLE",{"data-svelte-h":!0}),r(Jl)!=="svelte-1oxtqlv"&&(Jl.innerHTML=ke),Ls=a(l),Tl=p(l,"P",{"data-svelte-h":!0}),r(Tl)!=="svelte-f1s8c3"&&(Tl.innerHTML=Qe),Ps=a(l),Os=p(l,"HR",{}),Ds=a(l),o(wl.$$.fragment,l),Ks=a(l),dl=p(l,"P",{"data-svelte-h":!0}),r(dl)!=="svelte-6b2drq"&&(dl.innerHTML=Ne),le=a(l),o(hl.$$.fragment,l),se=a(l),bl=p(l,"P",{"data-svelte-h":!0}),r(bl)!=="svelte-18tauj5"&&(bl.innerHTML=_e),ee=a(l),Il=p(l,"P",{"data-svelte-h":!0}),r(Il)!=="svelte-r3ulln"&&(Il.innerHTML=Ve),te=a(l),gl=p(l,"UL",{"data-svelte-h":!0}),r(gl)!=="svelte-iyl68d"&&(gl.innerHTML=We),ne=a(l),o(fl.$$.fragment,l),ae=a(l),Al=p(l,"P",{}),$e(Al).forEach(e),this.h()},h(){pe(J,"name","hf:doc:metadata"),pe(J,"content",Oe),pe(T,"class","note")},m(l,s){xe(document.head,J),t(l,Bl,s),t(l,Cl,s),t(l,Zl,s),y(w,l,s),t(l,El,s),y(d,l,s),t(l,Gl,s),t(l,h,s),t(l,vl,s),t(l,b,s),t(l,Rl,s),t(l,I,s),t(l,kl,s),t(l,g,s),t(l,Ql,s),t(l,f,s),t(l,Nl,s),y(C,l,s),t(l,_l,s),t(l,A,s),t(l,Vl,s),t(l,B,s),t(l,Wl,s),y(Z,l,s),t(l,$l,s),t(l,E,s),t(l,Xl,s),t(l,Sl,s),t(l,Fl,s),y(G,l,s),t(l,ql,s),y(v,l,s),t(l,zl,s),t(l,Yl,s),t(l,xl,s),y(R,l,s),t(l,Hl,s),y(k,l,s),t(l,Ll,s),t(l,Q,s),t(l,Pl,s),y(N,l,s),t(l,Ol,s),y(_,l,s),t(l,Dl,s),t(l,Kl,s),t(l,ls,s),y(V,l,s),t(l,ss,s),t(l,W,s),t(l,es,s),y($,l,s),t(l,ts,s),t(l,ns,s),t(l,as,s),y(X,l,s),t(l,Ms,s),t(l,S,s),t(l,ps,s),t(l,F,s),t(l,is,s),y(q,l,s),t(l,os,s),t(l,rs,s),t(l,ys,s),y(z,l,s),t(l,cs,s),t(l,Y,s),t(l,js,s),y(x,l,s),t(l,Us,s),t(l,H,s),t(l,ms,s),y(L,l,s),t(l,us,s),t(l,P,s),t(l,Js,s),y(O,l,s),t(l,Ts,s),t(l,ws,s),t(l,ds,s),y(D,l,s),t(l,hs,s),t(l,K,s),t(l,bs,s),y(ll,l,s),t(l,Is,s),t(l,sl,s),t(l,gs,s),t(l,fs,s),t(l,Cs,s),y(el,l,s),t(l,As,s),t(l,tl,s),t(l,Bs,s),y(nl,l,s),t(l,Zs,s),t(l,al,s),t(l,Es,s),t(l,Ml,s),t(l,Gs,s),t(l,vs,s),t(l,Rs,s),y(pl,l,s),t(l,ks,s),t(l,il,s),t(l,Qs,s),y(ol,l,s),t(l,Ns,s),t(l,_s,s),t(l,Vs,s),y(rl,l,s),t(l,Ws,s),t(l,yl,s),t(l,$s,s),y(cl,l,s),t(l,Xs,s),t(l,T,s),t(l,Ss,s),t(l,Fs,s),t(l,qs,s),y(jl,l,s),t(l,zs,s),t(l,Ul,s),t(l,Ys,s),y(ml,l,s),t(l,xs,s),t(l,ul,s),t(l,Hs,s),t(l,Jl,s),t(l,Ls,s),t(l,Tl,s),t(l,Ps,s),t(l,Os,s),t(l,Ds,s),y(wl,l,s),t(l,Ks,s),t(l,dl,s),t(l,le,s),y(hl,l,s),t(l,se,s),t(l,bl,s),t(l,ee,s),t(l,Il,s),t(l,te,s),t(l,gl,s),t(l,ne,s),y(fl,l,s),t(l,ae,s),t(l,Al,s),Me=!0},p:Se,i(l){Me||(c(w.$$.fragment,l),c(d.$$.fragment,l),c(C.$$.fragment,l),c(Z.$$.fragment,l),c(G.$$.fragment,l),c(v.$$.fragment,l),c(R.$$.fragment,l),c(k.$$.fragment,l),c(N.$$.fragment,l),c(_.$$.fragment,l),c(V.$$.fragment,l),c($.$$.fragment,l),c(X.$$.fragment,l),c(q.$$.fragment,l),c(z.$$.fragment,l),c(x.$$.fragment,l),c(L.$$.fragment,l),c(O.$$.fragment,l),c(D.$$.fragment,l),c(ll.$$.fragment,l),c(el.$$.fragment,l),c(nl.$$.fragment,l),c(pl.$$.fragment,l),c(ol.$$.fragment,l),c(rl.$$.fragment,l),c(cl.$$.fragment,l),c(jl.$$.fragment,l),c(ml.$$.fragment,l),c(wl.$$.fragment,l),c(hl.$$.fragment,l),c(fl.$$.fragment,l),Me=!0)},o(l){j(w.$$.fragment,l),j(d.$$.fragment,l),j(C.$$.fragment,l),j(Z.$$.fragment,l),j(G.$$.fragment,l),j(v.$$.fragment,l),j(R.$$.fragment,l),j(k.$$.fragment,l),j(N.$$.fragment,l),j(_.$$.fragment,l),j(V.$$.fragment,l),j($.$$.fragment,l),j(X.$$.fragment,l),j(q.$$.fragment,l),j(z.$$.fragment,l),j(x.$$.fragment,l),j(L.$$.fragment,l),j(O.$$.fragment,l),j(D.$$.fragment,l),j(ll.$$.fragment,l),j(el.$$.fragment,l),j(nl.$$.fragment,l),j(pl.$$.fragment,l),j(ol.$$.fragment,l),j(rl.$$.fragment,l),j(cl.$$.fragment,l),j(jl.$$.fragment,l),j(ml.$$.fragment,l),j(wl.$$.fragment,l),j(hl.$$.fragment,l),j(fl.$$.fragment,l),Me=!1},d(l){l&&(e(Bl),e(Cl),e(Zl),e(El),e(Gl),e(h),e(vl),e(b),e(Rl),e(I),e(kl),e(g),e(Ql),e(f),e(Nl),e(_l),e(A),e(Vl),e(B),e(Wl),e($l),e(E),e(Xl),e(Sl),e(Fl),e(ql),e(zl),e(Yl),e(xl),e(Hl),e(Ll),e(Q),e(Pl),e(Ol),e(Dl),e(Kl),e(ls),e(ss),e(W),e(es),e(ts),e(ns),e(as),e(Ms),e(S),e(ps),e(F),e(is),e(os),e(rs),e(ys),e(cs),e(Y),e(js),e(Us),e(H),e(ms),e(us),e(P),e(Js),e(Ts),e(ws),e(ds),e(hs),e(K),e(bs),e(Is),e(sl),e(gs),e(fs),e(Cs),e(As),e(tl),e(Bs),e(Zs),e(al),e(Es),e(Ml),e(Gs),e(vs),e(Rs),e(ks),e(il),e(Qs),e(Ns),e(_s),e(Vs),e(Ws),e(yl),e($s),e(Xs),e(T),e(Ss),e(Fs),e(qs),e(zs),e(Ul),e(Ys),e(xs),e(ul),e(Hs),e(Jl),e(Ls),e(Tl),e(Ps),e(Os),e(Ds),e(Ks),e(dl),e(le),e(se),e(bl),e(ee),e(Il),e(te),e(gl),e(ne),e(ae),e(Al)),e(J),U(w,l),U(d,l),U(C,l),U(Z,l),U(G,l),U(v,l),U(R,l),U(k,l),U(N,l),U(_,l),U(V,l),U($,l),U(X,l),U(q,l),U(z,l),U(x,l),U(L,l),U(O,l),U(D,l),U(ll,l),U(el,l),U(nl,l),U(pl,l),U(ol,l),U(rl,l),U(cl,l),U(jl,l),U(ml,l),U(wl,l),U(hl,l),U(fl,l)}}}const Oe='{"title":"Collecting rollouts with OpenEnv for supervised training","local":"collecting-rollouts-with-openenv-for-supervised-training","sections":[{"title":"Why use an environment to collect training data","local":"why-use-an-environment-to-collect-training-data","sections":[],"depth":2},{"title":"What you’ll use","local":"what-youll-use","sections":[],"depth":2},{"title":"1. Install dependencies","local":"1-install-dependencies","sections":[],"depth":2},{"title":"2. Set your credentials","local":"2-set-your-credentials","sections":[],"depth":2},{"title":"3. Define the system prompt","local":"3-define-the-system-prompt","sections":[],"depth":2},{"title":"4. Configure data collection","local":"4-configure-data-collection","sections":[],"depth":2},{"title":"5. Collect rollouts with openenv collect","local":"5-collect-rollouts-with-openenv-collect","sections":[],"depth":2},{"title":"6. Filter the dataset","local":"6-filter-the-dataset","sections":[],"depth":2},{"title":"7. Inspect the dataset before training","local":"7-inspect-the-dataset-before-training","sections":[],"depth":2},{"title":"8. Measure token lengths","local":"8-measure-token-lengths","sections":[],"depth":2},{"title":"9. Fine-tune with SFTTrainer","local":"9-fine-tune-with-sfttrainer","sections":[],"depth":2},{"title":"10. Evaluate: before vs after","local":"10-evaluate-before-vs-after","sections":[],"depth":2},{"title":"11. Where to go next: GRPO","local":"11-where-to-go-next-grpo","sections":[],"depth":2}],"depth":1}';function De(ie){return Fe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class tt extends qe{constructor(J){super(),ze(this,J,De,Pe,Xe,{})}}export{tt as component};