openenv/pr_833/en/_app/immutable/nodes/56.05ca9c19.js

import{s as Fl,n as zl,o as Yl}from"../chunks/scheduler.2b22cead.js";import{S as xl,i as Ll,e as i,s as n,c as M,h as ql,a as o,d as l,b as a,f as Xl,g as p,j as r,k as Ce,l as Pl,m as s,n as c,t as w,o as d,p as h}from"../chunks/index.1a0e8013.js";import{C as Ol,H as y,E as Dl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.093aec8d.js";import{C as m}from"../chunks/CodeBlock.cbb387ef.js";function Kl(rl){let T,Ae,Be,Ge,f,_e,g,Ze,I,Ml='<a href="https://colab.research.google.com/github/huggingface/OpenEnv/blob/main/examples/end_to_end_walkthrough.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>',Re,b,pl="In this tutorial you’ll take a small open-weight model, an OpenEnv environment, and TRL, and run the full training pipeline end-to-end:",$e,C,cl="<li>Connect to a hosted environment.</li> <li>Wire it into TRL via the <code>environment_factory</code> pattern.</li> <li>Fine-tune with <strong>GRPO</strong> (Group Relative Policy Optimization).</li> <li>Read the reward delta from the training logs to see how much the policy improved.</li> <li>Publish the trained model to the Hub.</li>",ke,B,wl="The goal is to see the whole pipeline as one coherent narrative — model, environment, training, metric — rather than three separate articles.",Ee,v,We,A,dl="We pair <strong>GRPO</strong> with a <strong>procedural</strong> task on purpose. GRPO is a value-free RL method that ranks several rollouts of the same prompt against each other, so the only signal it needs is a per-rollout scalar reward — exactly what an environment can return after a <code>step</code>. Procedural means the env generates a fresh question every episode rather than serving a fixed dataset, so the model has to <em>generalize</em> over the family of problems instead of memorizing specific items.",Qe,G,Se,_,hl='<li><strong>Model</strong>: <a href="https://huggingface.co/Qwen/Qwen3-1.7B" rel="nofollow"><code>Qwen/Qwen3-1.7B</code></a> — fits a single A100 (40 GB) at the settings below and is large enough for GRPO to move the needle. For smaller GPUs, swap to <a href="https://huggingface.co/Qwen/Qwen3-0.6B" rel="nofollow"><code>Qwen/Qwen3-0.6B</code></a>.</li> <li><strong>Environment</strong>: <a href="https://github.com/huggingface/OpenEnv/tree/main/envs/reasoning_gym_env" rel="nofollow"><code>reasoning_gym_env</code></a>, an OpenEnv wrapper around the <a href="https://github.com/open-thought/reasoning-gym" rel="nofollow">Reasoning Gym</a> library. Each episode is a single Q→A.</li> <li><strong>Dataset</strong>: <code>chain_sum</code> from Reasoning Gym — chains of integer additions like <code>Compute 17 + 4 + 22 + 9</code>. Procedurally generated, so every rollout sees a fresh problem.</li> <li><strong>Trainer</strong>: <a href="https://huggingface.co/docs/trl/main/en/grpo_trainer" rel="nofollow">TRL <code>GRPOTrainer</code></a> with <code>environment_factory</code>.</li>',He,u,yl="<p>This tutorial runs through training; on a single A100 (40 GB) the recipe completes in roughly an hour at the suggested settings, peaking around ~38 GB of VRAM. T4 (16 GB) won’t fit Qwen3-1.7B at these settings — see the model bullet above for the smaller-GPU swap. The exact reward numbers you see will vary with seed and budget — the point is to watch the reward curve climb and report the delta.</p>",Ne,Ve,Xe,Z,Fe,R,ml='This tutorial connects to <a href="https://huggingface.co/spaces/sergiopaniego/reasoning_gym" rel="nofollow"><code>sergiopaniego/reasoning_gym</code></a>. For your own training runs, deploy your own copy first by running <code>openenv push --repo-id &lt;your-username&gt;/reasoning_gym</code> inside <code>envs/reasoning_gym_env/</code> of the OpenEnv repo, then replace <code>sergiopaniego</code> with your username in the install line and the <code>base_url=</code> strings further down.',ze,$,Tl="Install pip dependencies — keep them as separate cells (don’t merge into one <code>pip install</code>):",Ye,k,xe,Le,qe,E,Pe,W,ul="You’ll need to be logged in to download the base model and (optionally) push the trained checkpoint.",Oe,Q,De,Ke,et,S,tt,H,Jl="The model will be asked to use a single tool, <code>answer</code>, to submit its final number. The prompt makes that explicit.",lt,N,st,nt,at,V,it,X,Ul="The <code>environment_factory</code> pattern asks for a Python class that the trainer can instantiate per rollout. It needs:",ot,F,jl="<li>An <code>__init__</code> that opens a connection to the underlying environment.</li> <li>A <code>reset(**kwargs)</code> method that starts a new episode and returns the initial observation as a string (the question, in our case).</li> <li>One or more <em>tool methods</em> — public methods with docstrings — that the trainer auto-discovers and exposes as tools to the model. Each call corresponds to one <code>env.step</code> on the underlying environment.</li>",rt,z,fl="Because Reasoning Gym episodes are <strong>single-step</strong> (one question → one answer → done), the wrapper is small.",Mt,Y,pt,J,gl="<p>Replace the <code>base_url</code> with your own deployment if you’ve pushed <code>reasoning_gym_env</code> to your own Space — the hosted versions have limited concurrency and are intended for tutorials and small experiments.</p>",ct,x,wt,L,Il="It helps to picture the runtime loop. At init the trainer creates <code>gradient_accumulation_steps × per_device_train_batch_size</code> instances of <code>ReasoningGymTrainEnv</code> — these stay alive across optimizer steps. Per generation batch it then does, <strong>for each instance in parallel</strong>:",dt,q,bl="<li><code>env.reset(**row)</code> — opens (or reuses) the WebSocket session and returns the question string.</li> <li>The model is conditioned on that question, generates <code>num_generations</code> candidate completions, and the trainer parses any <code>&lt;tool_call&gt;</code> blocks out of each.</li> <li>For each parsed call it dispatches to the matching tool method (here, <code>answer(...)</code>) and feeds the return value back to the model as a <code>&lt;tool_response&gt;</code>.</li> <li>When the env signals <code>done=True</code>, the rollout ends and the trainer reads <code>env.reward</code>.</li> <li>GRPO computes one advantage per completion (relative to the group’s mean reward), updates the policy, and the cycle repeats.</li>",ht,P,Cl="That’s why the wrapper only needs three things — a connection in <code>__init__</code>, a <code>reset</code> that returns the initial obs, and one or more tool methods that update <code>self.reward</code>/<code>self.done</code>.",yt,mt,Tt,O,ut,D,Bl="The reward function receives the list of environment instances after each rollout. Each instance already tracks its own reward (set inside <code>answer()</code>), so we just read it back.",Jt,K,Ut,jt,ft,ee,gt,te,vl="Each row in the training dataset triggers one rollout episode. The prompt is identical across rows because the <em>environment</em> supplies the per-episode question — we’re using the dataset purely to control how many episodes the trainer runs.",It,le,bt,Ct,Bt,se,vt,ne,Al='These settings mirror the <a href="wordle-grpo">Wordle GRPO tutorial</a> and are tuned for a single A100 (40 GB). Bigger GPUs can raise <code>per_device_train_batch_size</code> and <code>num_generations</code>; smaller GPUs should drop to Qwen3-0.6B and shrink <code>max_completion_length</code>.',At,ae,Gt,ie,Gl="A few of the choices above worth flagging: <code>max_steps=150</code> caps the run before saturation (see <em>Reading the dashboard</em> below). <code>gradient_accumulation_steps=4</code> keeps the parallel env count at <code>1 × 4 = 4</code>, well under the server’s default concurrency limit. <code>save_strategy=&quot;no&quot;</code> skips intermediate checkpoints so the run stays quiet — we push the final model explicitly in section 9. <code>use_vllm</code> is left at its default (<code>False</code>); enabling it speeds up rollouts on bare-metal but its distributed init breaks under IPython.",_t,U,_l="<p><code>chat_template_kwargs={&quot;enable_thinking&quot;: False}</code> disables Qwen3’s thinking mode so the model emits tool calls directly instead of reasoning tokens first. For a pure tool-use task like this one that’s what you want; for harder math you may benefit from re-enabling it and growing <code>max_completion_length</code>.</p>",Zt,Rt,$t,oe,kt,re,Zl="<code>environment_factory=ReasoningGymTrainEnv</code> is the only piece wiring our wrapper into the training loop.",Et,Me,Wt,pe,Qt,ce,Rl="Open the Trackio Space linked in the trainer logs to follow the run live. A healthy GRPO trajectory looks roughly like this:",St,we,$l="<li><strong><code>reward</code></strong> climbs from your baseline toward <code>1.0</code> over the first ~100 steps. A flat line near 0 means the task is too hard for the base model; a flat line near 1 means it’s too easy — adjust <code>DATASET_CONFIG</code> in either case.</li> <li><strong><code>reward_std</code></strong> starts moderate and <em>drops</em> as the policy converges (most rollouts succeed). Persistent zero means every rollout in the group gives the same score → no advantage signal → no learning. Bump <code>num_generations</code> or task difficulty.</li> <li><strong><code>frac_reward_zero_std</code></strong> is the fraction of groups where every rollout has the same reward — when it climbs toward 1.0 you’ve saturated.</li> <li><strong><code>entropy</code></strong> stays low while the model is learning. Once <code>reward</code> saturates, <code>entropy</code> typically rises again because the policy gradient is zero and only the KL penalty against the reference model is active — at that point further training is net-negative. Stop with a kernel interrupt or trust <code>max_steps</code>.</li> <li><strong><code>grad_norm</code></strong> decays toward zero as gradients become uninformative; same saturation signal.</li>",Ht,de,kl="Once training finishes, the model in the running process has been fine-tuned in place.",Nt,Vt,Xt,he,Ft,ye,El="<code>save_strategy=&quot;no&quot;</code> means the trainer didn’t write any intermediate checkpoints. Push the final model explicitly so others can reuse it (and so the experiment is reproducible from the Hub):",zt,me,Yt,Te,Wl="The repo is derived automatically from <code>output_dir</code> (or <code>hub_model_id</code> if set in <code>GRPOConfig</code>). After this completes, the model lives at <code>https://huggingface.co/&lt;your-username&gt;/reasoning-gym-chain-sum-Qwen3-1.7B</code> and anyone can load it with <code>AutoModelForCausalLM.from_pretrained(...)</code>.",xt,Lt,qt,ue,Pt,Je,Ql="Every rollout the trainer ran left a <code>reward</code> entry in <code>trainer.state.log_history</code>. Comparing the first few logged rewards (the model’s starting capability) to the last few (after training) gives a clean before/after number — same metric, same distribution, no second eval pass required.",Ot,Ue,Dt,je,Sl="A delta of <strong>+10 to +30 pp</strong> is what you should expect at this difficulty; outside that range:",Kt,fe,Hl='<li><strong>Δ ≈ 0 pp, initial already high (≥90%)</strong> — <code>DATASET_CONFIG</code> is too easy; the model already solves it before training. Bump <code>min_terms</code> / <code>min_digits</code>.</li> <li><strong>Δ ≈ 0 pp, initial very low (≤20%)</strong> — task is too hard for the base model to ever stumble onto a correct answer, so GRPO has no positive rollouts to learn from. Lower <code>min_terms</code> / <code>min_digits</code>. If the reward stays near zero even at minimum difficulty, the bottleneck is likely <strong>format compliance</strong> rather than task difficulty — the model never produces a valid <code>&lt;tool_call&gt;</code> so the env cannot score it. See the <a href="sft-warmup">SFT warm-up tutorial</a> for how to fix this before returning to GRPO.</li> <li><strong>Δ negative</strong> — you trained past saturation: once <code>reward</code> plateaus, the KL penalty starts pulling the policy back toward the reference. Reduce <code>max_steps</code> so training stops while it’s still net-improving.</li>',el,j,Nl='<p>This delta is measured <em>during training</em> — same prompt format, same env, same procedural distribution that produced each rollout. It’s the most direct way to ask “did the policy improve over the run?“. A more rigorous protocol — generating completions on a held-out split with a separate evaluation harness — is what frameworks like <a href="https://inspect.aisi.org.uk/" rel="nofollow">Inspect AI</a> are designed for; that’s a follow-up rather than part of this walkthrough.</p>',tl,ll,sl,ge,nl,Ie,Vl='<li><strong>Swap the dataset.</strong> <code>chain_sum</code> is one of ~100 datasets in <a href="https://github.com/open-thought/reasoning-gym" rel="nofollow">Reasoning Gym</a> — try <code>simple_equations</code>, <code>letter_counting</code>, or <code>propositional_logic</code> by changing <code>DATASET_NAME</code> and re-running the same recipe.</li> <li><strong>Try a different environment.</strong> The same <code>environment_factory</code> shape works for any OpenEnv environment with a small tool surface — browse the <a href="../environments">environment catalog</a> for ideas.</li> <li><strong>Use SFT as a warm-start.</strong> If format compliance is the bottleneck (initial reward near zero regardless of difficulty), the <a href="sft-warmup">SFT warm-up tutorial</a> shows how to collect teacher rollouts, filter by reward, and fine-tune a student model — so GRPO starts with non-zero <code>reward_std</code> from the first batch.</li> <li><strong>Read the other tutorials.</strong> <a href="wordle-grpo">Wordle GRPO</a> covers the multi-step variant; the full list is in the <a href="index">tutorials index</a>.</li>',al,be,il,ve,ol;return f=new Ol({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),g=new y({props:{title:"End-to-end OpenEnv walkthrough: train a reasoning agent with GRPO",local:"end-to-end-openenv-walkthrough-train-a-reasoning-agent-with-grpo",headingTag:"h1"}}),v=new y({props:{title:"Why this shape",local:"why-this-shape",headingTag:"h2"}}),G=new y({props:{title:"What you’ll use",local:"what-youll-use",headingTag:"h2"}}),Z=new y({props:{title:"1. Install dependencies",local:"1-install-dependencies",headingTag:"h2"}}),k=new m({props:{code:"IXBpcCUyMGluc3RhbGwlMjAtcSUyMHRybCUwQSFwaXAlMjBpbnN0YWxsJTIwLXElMjBvcGVuZW52JTBBIXBpcCUyMGluc3RhbGwlMjAtcSUyMC0tbm8tZGVwcyUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZzcGFjZXMlMkZzZXJnaW9wYW5pZWdvJTJGcmVhc29uaW5nX2d5bSUwQSFwaXAlMjBpbnN0YWxsJTIwLVVxJTIwJTIydHJhbnNmb3JtZXJzJTNFJTNENS4zLjAlMjIlMjAlMjAlMjMlMjA1LjMlMkIlMjBoYXMlMjB0aGUlMjAlNjBlbnZpcm9ubWVudF9mYWN0b3J5JTYwJTIwaW50ZWdyYXRpb24lMjBUUkwlMjBuZWVkcyUwQSFwaXAlMjBpbnN0YWxsJTIwLXElMjB0cmFja2lvJTIwam1lc3BhdGg=",highlighted:`!pip install -q trl
!pip install -q openenv
!pip install -q --no-deps git+https://huggingface.co/spaces/sergiopaniego/reasoning_gym
!pip install -Uq <span class="hljs-string">&quot;transformers&gt;=5.3.0&quot;</span>  <span class="hljs-comment"># 5.3+ has the \`environment_factory\` integration TRL needs</span>
!pip install -q trackio jmespath`,lang:"python",wrap:!1}}),E=new y({props:{title:"2. Log in to Hugging Face",local:"2-log-in-to-hugging-face",headingTag:"h2"}}),Q=new m({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,lang:"python",wrap:!1}}),S=new y({props:{title:"3. Define the system prompt",local:"3-define-the-system-prompt",headingTag:"h2"}}),N=new m({props:{code:"cHJvbXB0JTIwJTNEJTIwJTIyJTIyJTIyWW91JTIwYXJlJTIwYSUyMGNhcmVmdWwlMjBhcml0aG1ldGljJTIwYXNzaXN0YW50LiUwQSUwQVlvdSUyMHdpbGwlMjBiZSUyMGdpdmVuJTIwYSUyMGNoYWluJTIwb2YlMjBpbnRlZ2VyJTIwYWRkaXRpb25zLiUyMENvbXB1dGUlMjB0aGUlMjByZXN1bHQlMjBhbmQlMjBzdWJtaXQlMjBpdCUyMGFzJTIwYSUyMHNpbmdsZSUyMG51bWJlci4lMEElMEFSdWxlcyUzQSUwQTEuJTIwUmVhZCUyMHRoZSUyMHF1ZXN0aW9uJTIwY2FyZWZ1bGx5LiUwQTIuJTIwVXNlJTIwdGhlJTIwdG9vbCUyMCU2MGFuc3dlciU2MCUyMGV4YWN0bHklMjBvbmNlJTIwd2l0aCUyMHlvdXIlMjBmaW5hbCUyMG51bWJlci4lMEEzLiUyMFRoZSUyMGFuc3dlciUyMG11c3QlMjBiZSUyMGElMjBzaW5nbGUlMjBpbnRlZ2VyJTIwd2l0aCUyMG5vJTIwdW5pdHMlMjBvciUyMGV4cGxhbmF0aW9uLiUwQSUyMiUyMiUyMg==",highlighted:`prompt = <span class="hljs-string">&quot;&quot;&quot;You are a careful arithmetic assistant.

You will be given a chain of integer additions. Compute the result and submit it as a single number.

Rules:
1. Read the question carefully.
2. Use the tool \`answer\` exactly once with your final number.
3. The answer must be a single integer with no units or explanation.
&quot;&quot;&quot;</span>`,lang:"python",wrap:!1}}),V=new y({props:{title:"4. Define the environment class",local:"4-define-the-environment-class",headingTag:"h2"}}),Y=new m({props:{code:"aW1wb3J0JTIwcmFuZG9tJTBBJTBBZnJvbSUyMHJlYXNvbmluZ19neW1fZW52JTIwaW1wb3J0JTIwUmVhc29uaW5nR3ltQWN0aW9uJTJDJTIwUmVhc29uaW5nR3ltRW52JTBBJTBBJTBBY2xhc3MlMjBSZWFzb25pbmdHeW1UcmFpbkVudiUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMkVudmlyb25tZW50JTIwd3JhcHBlciUyMGZvciUyMEdSUE8lMjB0cmFpbmluZyUyMG9uJTIwY2hhaW5fc3VtLiUwQSUwQSUyMCUyMCUyMCUyMEVhY2glMjByb2xsb3V0JTIwZXBpc29kZSUyMCUzRCUyMG9uZSUyMHF1ZXN0aW9uJTIwJUUyJTg2JTkyJTIwb25lJTIwJTYwYW5zd2VyJTYwJTIwdG9vbCUyMGNhbGwlMjAlRTIlODYlOTIlMjBkb25lLiUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMiUwQSUwQSUyMCUyMCUyMCUyMERBVEFTRVRfTkFNRSUyMCUzRCUyMCUyMmNoYWluX3N1bSUyMiUwQSUyMCUyMCUyMCUyMERBVEFTRVRfU0laRSUyMCUzRCUyMDEwMDAlMEElMjAlMjAlMjAlMjBEQVRBU0VUX0NPTkZJRyUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1pbl90ZXJtcyUyMiUzQSUyMDIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJtYXhfdGVybXMlMjIlM0ElMjAzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybWluX2RpZ2l0cyUyMiUzQSUyMDIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJtYXhfZGlnaXRzJTIyJTNBJTIwMiUyQyUwQSUyMCUyMCUyMCUyMCU3RCUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwJTYwRW52Q2xpZW50JTYwJTIwc3ViY2xhc3NlcyUyMGFyZSUyMGFzeW5jJTIwYnklMjBkZWZhdWx0JTNCJTIwJTYwLnN5bmMoKSU2MCUyMHJldHVybnMlMjBhJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwc3luY2hyb25vdXMlMjB3cmFwcGVyJTIwc28lMjB0aGUlMjB0cmFpbmVyJTIwY2FuJTIwY2FsbCUyMG91ciUyMHRvb2wlMjBtZXRob2RzJTIwZGlyZWN0bHkuJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5jbGllbnQlMjAlM0QlMjBSZWFzb25pbmdHeW1FbnYoYmFzZV91cmwlM0QlMjJodHRwcyUzQSUyRiUyRnNlcmdpb3BhbmllZ28tcmVhc29uaW5nLWd5bS5oZi5zcGFjZSUyMikuc3luYygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwUmFuZG9tJTIwc2VlZCUyMHBlciUyMGluc3RhbmNlJTIwc28lMjB0aGUlMjBwYXJhbGxlbCUyMGVudnMlMjB0aGUlMjB0cmFpbmVyJTIwY3JlYXRlcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGRvbid0JTIwYWxsJTIwaXRlcmF0ZSUyMG92ZXIlMjB0aGUlMjBzYW1lJTIwcXVlc3Rpb24lMjBzZXF1ZW5jZS4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLl9kYXRhc2V0X3NlZWQlMjAlM0QlMjByYW5kb20ucmFuZGludCgwJTJDJTIwMioqMzElMjAtJTIwMSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLl9pbml0aWFsaXplZCUyMCUzRCUyMEZhbHNlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5yZXdhcmQlMjAlM0QlMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmRvbmUlMjAlM0QlMjBGYWxzZSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMHJlc2V0KHNlbGYlMkMlMjAqKmt3YXJncyklMjAtJTNFJTIwc3RyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBub3QlMjBzZWxmLl9pbml0aWFsaXplZCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEZpcnN0JTIwcmVzZXQlM0ElMjBjb25maWd1cmUlMjB0aGUlMjBkYXRhc2V0JTIwKG5hbWUlMjAlMkIlMjBjb25maWclMjAlMkIlMjBzZWVkJTIwJTJCJTIwc2l6ZSkuJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwc2VsZi5jbGllbnQucmVzZXQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGF0YXNldF9uYW1lJTNEc2VsZi5EQVRBU0VUX05BTUUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkYXRhc2V0X2NvbmZpZyUzRHNlbGYuREFUQVNFVF9DT05GSUclMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWVkJTNEc2VsZi5fZGF0YXNldF9zZWVkJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2l6ZSUzRHNlbGYuREFUQVNFVF9TSVpFJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuX2luaXRpYWxpemVkJTIwJTNEJTIwVHJ1ZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBTdWJzZXF1ZW50JTIwcmVzZXRzJTNBJTIwbm8lMjBhcmdzJTIwJUUyJTg2JTkyJTIwc2VydmVyJTIwcmV0dXJucyUyMHRoZSUyMG5leHQlMjBxdWVzdGlvbiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGZyb20lMjB0aGUlMjBzYW1lJTIwZGF0YXNldCUyMGl0ZXJhdG9yLiUyMFJlLXNlbmRpbmclMjB0aGUlMjBjb25maWclMjB3b3VsZCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMHJlYnVpbGQlMjB0aGUlMjBkYXRhc2V0JTIwYW5kJTIwcmV3aW5kJTIwdG8lMjBxdWVzdGlvbiUyMDAuJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwc2VsZi5jbGllbnQucmVzZXQoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYucmV3YXJkJTIwJTNEJTIwMC4wJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5kb25lJTIwJTNEJTIwRmFsc2UlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjByZXN1bHQub2JzZXJ2YXRpb24ucXVlc3Rpb24lMEElMEElMjAlMjAlMjAlMjBkZWYlMjBhbnN3ZXIoc2VsZiUyQyUyMGFuc3dlciUzQSUyMHN0ciklMjAtJTNFJTIwc3RyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyJTIyJTIyU3VibWl0JTIwdGhlJTIwZmluYWwlMjBhbnN3ZXIlMjBmb3IlMjB0aGUlMjBjdXJyZW50JTIwcXVlc3Rpb24uJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwQXJncyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFuc3dlciUzQSUyMFRoZSUyMGFnZW50J3MlMjBhbnN3ZXIlMjAod2lsbCUyMGJlJTIwcGFyc2VkJTIwYXMlMjBhJTIwbnVtYmVyJTIwc2VydmVyLXNpZGUpLiUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFJldHVybnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBBJTIwc2hvcnQlMjBmZWVkYmFjayUyMHN0cmluZyUyMHdpdGglMjB0aGUlMjBzY29yZSUyMGFuZCUyMHRoZSUyMGNvcnJlY3QlMjBhbnN3ZXIuJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBzZWxmLmRvbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByYWlzZSUyMFZhbHVlRXJyb3IoJTIyRXBpc29kZSUyMGlzJTIwYWxyZWFkeSUyMGZpbmlzaGVkLiUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBUaGUlMjBtb2RlbCUyMG9mdGVuJTIwZW1pdHMlMjAlNjBhbnN3ZXIlNjAlMjBhcyUyMGElMjBKU09OJTIwaW50JTIwKGUuZy4lMjAlNjA3JTYwKSUyMGV2ZW4lMjB0aG91Z2glMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjB0aGUlMjB0b29sJTIwc2NoZW1hJTIwZGVjbGFyZXMlMjBzdHJpbmclMjAlRTIlODAlOTQlMjBjb2VyY2UlMjBzbyUyMHB5ZGFudGljJTIwdmFsaWRhdGlvbiUyMG9uJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwJTYwUmVhc29uaW5nR3ltQWN0aW9uJTYwJTIwZG9lc24ndCUyMHJlamVjdCUyMHRoZSUyMHJvbGxvdXQuJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwc2VsZi5jbGllbnQuc3RlcChSZWFzb25pbmdHeW1BY3Rpb24oYW5zd2VyJTNEc3RyKGFuc3dlcikpKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYucmV3YXJkJTIwJTNEJTIwZmxvYXQocmVzdWx0Lm9ic2VydmF0aW9uLnNjb3JlJTIwb3IlMjAwLjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5kb25lJTIwJTNEJTIwVHJ1ZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMGYlMjJzY29yZSUzRCU3QnNlbGYucmV3YXJkJTdEJTIwY29ycmVjdCUzRCU3QnJlc3VsdC5vYnNlcnZhdGlvbi5jb3JyZWN0X2Fuc3dlciU3RCUyMg==",highlighted:`<span class="hljs-keyword">import</span> random

<span class="hljs-keyword">from</span> reasoning_gym_env <span class="hljs-keyword">import</span> ReasoningGymAction, ReasoningGymEnv


<span class="hljs-keyword">class</span> <span class="hljs-title class_">ReasoningGymTrainEnv</span>:
    <span class="hljs-string">&quot;&quot;&quot;Environment wrapper for GRPO training on chain_sum.

    Each rollout episode = one question → one \`answer\` tool call → done.
    &quot;&quot;&quot;</span>

    DATASET_NAME = <span class="hljs-string">&quot;chain_sum&quot;</span>
    DATASET_SIZE = <span class="hljs-number">1000</span>
    DATASET_CONFIG = {
        <span class="hljs-string">&quot;min_terms&quot;</span>: <span class="hljs-number">2</span>,
        <span class="hljs-string">&quot;max_terms&quot;</span>: <span class="hljs-number">3</span>,
        <span class="hljs-string">&quot;min_digits&quot;</span>: <span class="hljs-number">2</span>,
        <span class="hljs-string">&quot;max_digits&quot;</span>: <span class="hljs-number">2</span>,
    }

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-comment"># \`EnvClient\` subclasses are async by default; \`.sync()\` returns a</span>
        <span class="hljs-comment"># synchronous wrapper so the trainer can call our tool methods directly.</span>
        self.client = ReasoningGymEnv(base_url=<span class="hljs-string">&quot;https://sergiopaniego-reasoning-gym.hf.space&quot;</span>).sync()
        <span class="hljs-comment"># Random seed per instance so the parallel envs the trainer creates</span>
        <span class="hljs-comment"># don&#x27;t all iterate over the same question sequence.</span>
        self._dataset_seed = random.randint(<span class="hljs-number">0</span>, <span class="hljs-number">2</span>**<span class="hljs-number">31</span> - <span class="hljs-number">1</span>)
        self._initialized = <span class="hljs-literal">False</span>
        self.reward = <span class="hljs-number">0.0</span>
        self.done = <span class="hljs-literal">False</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, **kwargs</span>) -&gt; <span class="hljs-built_in">str</span>:
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> self._initialized:
            <span class="hljs-comment"># First reset: configure the dataset (name + config + seed + size).</span>
            result = self.client.reset(
                dataset_name=self.DATASET_NAME,
                dataset_config=self.DATASET_CONFIG,
                seed=self._dataset_seed,
                size=self.DATASET_SIZE,
            )
            self._initialized = <span class="hljs-literal">True</span>
        <span class="hljs-keyword">else</span>:
            <span class="hljs-comment"># Subsequent resets: no args → server returns the next question</span>
            <span class="hljs-comment"># from the same dataset iterator. Re-sending the config would</span>
            <span class="hljs-comment"># rebuild the dataset and rewind to question 0.</span>
            result = self.client.reset()
        self.reward = <span class="hljs-number">0.0</span>
        self.done = <span class="hljs-literal">False</span>
        <span class="hljs-keyword">return</span> result.observation.question

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">answer</span>(<span class="hljs-params">self, answer: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span>:
        <span class="hljs-string">&quot;&quot;&quot;Submit the final answer for the current question.

        Args:
            answer: The agent&#x27;s answer (will be parsed as a number server-side).

        Returns:
            A short feedback string with the score and the correct answer.
        &quot;&quot;&quot;</span>
        <span class="hljs-keyword">if</span> self.done:
            <span class="hljs-keyword">raise</span> ValueError(<span class="hljs-string">&quot;Episode is already finished.&quot;</span>)
        <span class="hljs-comment"># The model often emits \`answer\` as a JSON int (e.g. \`7\`) even though</span>
        <span class="hljs-comment"># the tool schema declares string — coerce so pydantic validation on</span>
        <span class="hljs-comment"># \`ReasoningGymAction\` doesn&#x27;t reject the rollout.</span>
        result = self.client.step(ReasoningGymAction(answer=<span class="hljs-built_in">str</span>(answer)))
        self.reward = <span class="hljs-built_in">float</span>(result.observation.score <span class="hljs-keyword">or</span> <span class="hljs-number">0.0</span>)
        self.done = <span class="hljs-literal">True</span>
        <span class="hljs-keyword">return</span> <span class="hljs-string">f&quot;score=<span class="hljs-subst">{self.reward}</span> correct=<span class="hljs-subst">{result.observation.correct_answer}</span>&quot;</span>`,lang:"python",wrap:!1}}),x=new y({props:{title:"What the trainer does with this class",local:"what-the-trainer-does-with-this-class",headingTag:"h3"}}),O=new y({props:{title:"5. Define the reward function",local:"5-define-the-reward-function",headingTag:"h2"}}),K=new m({props:{code:"ZGVmJTIwcmV3YXJkX2Z1bmMoZW52aXJvbm1lbnRzJTJDJTIwKiprd2FyZ3MpJTIwLSUzRSUyMGxpc3QlNUJmbG9hdCU1RCUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QmVudi5yZXdhcmQlMjBmb3IlMjBlbnYlMjBpbiUyMGVudmlyb25tZW50cyU1RA==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">environments, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
    <span class="hljs-keyword">return</span> [env.reward <span class="hljs-keyword">for</span> env <span class="hljs-keyword">in</span> environments]`,lang:"python",wrap:!1}}),ee=new y({props:{title:"6. Create the dataset",local:"6-create-the-dataset",headingTag:"h2"}}),le=new m({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQSUwQWRhdGFzZXQlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglMEElMjAlMjAlMjAlMjAlN0IlMjJwcm9tcHQlMjIlM0ElMjAlNUIlNUIlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjBwcm9tcHQlN0QlNUQlMjBmb3IlMjBfJTIwaW4lMjByYW5nZSgxMDAwKSU1RCU3RCUwQSk=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

dataset = Dataset.from_dict(
    {<span class="hljs-string">&quot;prompt&quot;</span>: [[{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt}] <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>)]}
)`,lang:"python",wrap:!1}}),se=new y({props:{title:"7. Set the GRPO config",local:"7-set-the-grpo-config",headingTag:"h2"}}),ae=new m({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMEElMEFvdXRwdXRfZGlyJTIwJTNEJTIwJTIycmVhc29uaW5nLWd5bS1jaGFpbi1zdW0tUXdlbjMtMS43QiUyMiUwQSUwQWdycG9fY29uZmlnJTIwJTNEJTIwR1JQT0NvbmZpZyglMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMSUyQyUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzRDE1MCUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QxZS02JTJDJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNENCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMEElMjAlMjAlMjAlMjB3YXJtdXBfc3RlcHMlM0QxMCUyQyUwQSUyMCUyMCUyMCUyMG9wdGltJTNEJTIyYWRhbXdfdG9yY2glMjIlMkMlMEElMjAlMjAlMjAlMjBtYXhfZ3JhZF9ub3JtJTNEMS4wJTJDJTBBJTIwJTIwJTIwJTIwbnVtX2dlbmVyYXRpb25zJTNEMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9jb21wbGV0aW9uX2xlbmd0aCUzRDI1NiUyQyUwQSUyMCUyMCUyMCUyMGxvZ19jb21wbGV0aW9ucyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBudW1fY29tcGxldGlvbnNfdG9fcHJpbnQlM0QyJTJDJTBBJTIwJTIwJTIwJTIwY2hhdF90ZW1wbGF0ZV9rd2FyZ3MlM0QlN0IlMjJlbmFibGVfdGhpbmtpbmclMjIlM0ElMjBGYWxzZSU3RCUyQyUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0RvdXRwdXRfZGlyJTJDJTBBJTIwJTIwJTIwJTIwcmVwb3J0X3RvJTNEJTIydHJhY2tpbyUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYWNraW9fc3BhY2VfaWQlM0RvdXRwdXRfZGlyJTJDJTBBJTIwJTIwJTIwJTIwbG9nZ2luZ19zdGVwcyUzRDEwJTJDJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfY2hlY2twb2ludGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0cmF0ZWd5JTNEJTIybm8lMjIlMkMlMEElMjAlMjAlMjAlMjBwdXNoX3RvX2h1YiUzRFRydWUlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig

output_dir = <span class="hljs-string">&quot;reasoning-gym-chain-sum-Qwen3-1.7B&quot;</span>

grpo_config = GRPOConfig(
    num_train_epochs=<span class="hljs-number">1</span>,
    max_steps=<span class="hljs-number">150</span>,
    learning_rate=<span class="hljs-number">1e-6</span>,
    gradient_accumulation_steps=<span class="hljs-number">4</span>,
    per_device_train_batch_size=<span class="hljs-number">1</span>,
    warmup_steps=<span class="hljs-number">10</span>,
    optim=<span class="hljs-string">&quot;adamw_torch&quot;</span>,
    max_grad_norm=<span class="hljs-number">1.0</span>,
    num_generations=<span class="hljs-number">2</span>,
    max_completion_length=<span class="hljs-number">256</span>,
    log_completions=<span class="hljs-literal">True</span>,
    num_completions_to_print=<span class="hljs-number">2</span>,
    chat_template_kwargs={<span class="hljs-string">&quot;enable_thinking&quot;</span>: <span class="hljs-literal">False</span>},
    output_dir=output_dir,
    report_to=<span class="hljs-string">&quot;trackio&quot;</span>,
    trackio_space_id=output_dir,
    logging_steps=<span class="hljs-number">10</span>,
    gradient_checkpointing=<span class="hljs-literal">True</span>,
    save_strategy=<span class="hljs-string">&quot;no&quot;</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
)`,lang:"python",wrap:!1}}),oe=new y({props:{title:"8. Create the GRPOTrainer and start training",local:"8-create-the-grpotrainer-and-start-training",headingTag:"h2"}}),Me=new m({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTBBJTBBTU9ERUxfTkFNRSUyMCUzRCUyMCUyMlF3ZW4lMkZRd2VuMy0xLjdCJTIyJTBBJTBBdHJhaW5lciUyMCUzRCUyMEdSUE9UcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNETU9ERUxfTkFNRSUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRHJld2FyZF9mdW5jJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEZ3Jwb19jb25maWclMkMlMEElMjAlMjAlMjAlMjBlbnZpcm9ubWVudF9mYWN0b3J5JTNEUmVhc29uaW5nR3ltVHJhaW5FbnYlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer

MODEL_NAME = <span class="hljs-string">&quot;Qwen/Qwen3-1.7B&quot;</span>

trainer = GRPOTrainer(
    model=MODEL_NAME,
    reward_funcs=reward_func,
    train_dataset=dataset,
    args=grpo_config,
    environment_factory=ReasoningGymTrainEnv,
)

trainer.train()`,lang:"python",wrap:!1}}),pe=new y({props:{title:"Reading the trackio dashboard while it runs",local:"reading-the-trackio-dashboard-while-it-runs",headingTag:"h3"}}),he=new y({props:{title:"9. Publish the trained model to the Hub",local:"9-publish-the-trained-model-to-the-hub",headingTag:"h2"}}),me=new m({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yihjb21taXRfbWVzc2FnZSUzRCUyMkdSUE8lMjBmaW5lLXR1bmUlMjBvbiUyMHJlYXNvbmluZ19neW0lMjBjaGFpbl9zdW0lMjIp",highlighted:'trainer.push_to_hub(commit_message=<span class="hljs-string">&quot;GRPO fine-tune on reasoning_gym chain_sum&quot;</span>)',lang:"python",wrap:!1}}),ue=new y({props:{title:"10. Read the training reward delta",local:"10-read-the-training-reward-delta",headingTag:"h2"}}),Ue=new m({props:{code:"aW1wb3J0JTIwc3RhdGlzdGljcyUwQSUwQXJld2FyZHMlMjAlM0QlMjAlNUJsb2clNUIlMjJyZXdhcmQlMjIlNUQlMjBmb3IlMjBsb2clMjBpbiUyMHRyYWluZXIuc3RhdGUubG9nX2hpc3RvcnklMjBpZiUyMCUyMnJld2FyZCUyMiUyMGluJTIwbG9nJTVEJTBBJTBBaWYlMjBsZW4ocmV3YXJkcyklMjAlM0MlMjA1JTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMk9ubHklMjAlN0JsZW4ocmV3YXJkcyklN0QlMjByZXdhcmQlMjBlbnRyaWVzJTIwbG9nZ2VkJTIwJUUyJTgwJTk0JTIwdHJhaW4lMjBmb3IlMjBhJTIwZmV3JTIwbW9yZSUyMCU2MGxvZ2dpbmdfc3RlcHMlNjAlMjBhbmQlMjByZS1ydW4uJTIyKSUwQWVsc2UlM0ElMEElMjAlMjAlMjAlMjBpbml0aWFsJTIwJTNEJTIwc3RhdGlzdGljcy5tZWFuKHJld2FyZHMlNUIlM0E1JTVEKSUwQSUyMCUyMCUyMCUyMGZpbmFsJTIwJTNEJTIwc3RhdGlzdGljcy5tZWFuKHJld2FyZHMlNUItNSUzQSU1RCklMEElMjAlMjAlMjAlMjBwcmludChmJTIySW5pdGlhbCUyMHJld2FyZCUyMChmaXJzdCUyMDUlMjBsb2dzJTIwYXZnKSUzQSUyMCU3QmluaXRpYWwlM0EuMiUyNSU3RCUyMiklMEElMjAlMjAlMjAlMjBwcmludChmJTIyRmluYWwlMjByZXdhcmQlMjAlMjAlMjAobGFzdCUyMDUlMjBsb2dzJTIwYXZnKSUzQSUyMCUyMCU3QmZpbmFsJTNBLjIlMjUlN0QlMjIpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMkRlbHRhJTNBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCKGZpbmFsJTIwLSUyMGluaXRpYWwpJTIwKiUyMDEwMCUzQSUyQi4yZiU3RCUyMHBwJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> statistics

rewards = [log[<span class="hljs-string">&quot;reward&quot;</span>] <span class="hljs-keyword">for</span> log <span class="hljs-keyword">in</span> trainer.state.log_history <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;reward&quot;</span> <span class="hljs-keyword">in</span> log]

<span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(rewards) &lt; <span class="hljs-number">5</span>:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Only <span class="hljs-subst">{<span class="hljs-built_in">len</span>(rewards)}</span> reward entries logged — train for a few more \`logging_steps\` and re-run.&quot;</span>)
<span class="hljs-keyword">else</span>:
    initial = statistics.mean(rewards[:<span class="hljs-number">5</span>])
    final = statistics.mean(rewards[-<span class="hljs-number">5</span>:])
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Initial reward (first 5 logs avg): <span class="hljs-subst">{initial:<span class="hljs-number">.2</span>%}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Final reward   (last 5 logs avg):  <span class="hljs-subst">{final:<span class="hljs-number">.2</span>%}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Delta:                             <span class="hljs-subst">{(final - initial) * <span class="hljs-number">100</span>:+<span class="hljs-number">.2</span>f}</span> pp&quot;</span>)`,lang:"python",wrap:!1}}),ge=new y({props:{title:"11. Where to go next",local:"11-where-to-go-next",headingTag:"h2"}}),be=new Dl({props:{source:"https://github.com/huggingface/openenv/blob/main/docs/source/tutorials/end-to-end-walkthrough.md"}}),{c(){T=i("meta"),Ae=n(),Be=i("p"),Ge=n(),M(f.$$.fragment),_e=n(),M(g.$$.fragment),Ze=n(),I=i("p"),I.innerHTML=Ml,Re=n(),b=i("p"),b.textContent=pl,$e=n(),C=i("ol"),C.innerHTML=cl,ke=n(),B=i("p"),B.textContent=wl,Ee=n(),M(v.$$.fragment),We=n(),A=i("p"),A.innerHTML=dl,Qe=n(),M(G.$$.fragment),Se=n(),_=i("ul"),_.innerHTML=hl,He=n(),u=i("blockquote"),u.innerHTML=yl,Ne=n(),Ve=i("hr"),Xe=n(),M(Z.$$.fragment),Fe=n(),R=i("p"),R.innerHTML=ml,ze=n(),$=i("p"),$.innerHTML=Tl,Ye=n(),M(k.$$.fragment),xe=n(),Le=i("hr"),qe=n(),M(E.$$.fragment),Pe=n(),W=i("p"),W.textContent=ul,Oe=n(),M(Q.$$.fragment),De=n(),Ke=i("hr"),et=n(),M(S.$$.fragment),tt=n(),H=i("p"),H.innerHTML=Jl,lt=n(),M(N.$$.fragment),st=n(),nt=i("hr"),at=n(),M(V.$$.fragment),it=n(),X=i("p"),X.innerHTML=Ul,ot=n(),F=i("ul"),F.innerHTML=jl,rt=n(),z=i("p"),z.innerHTML=fl,Mt=n(),M(Y.$$.fragment),pt=n(),J=i("blockquote"),J.innerHTML=gl,ct=n(),M(x.$$.fragment),wt=n(),L=i("p"),L.innerHTML=Il,dt=n(),q=i("ol"),q.innerHTML=bl,ht=n(),P=i("p"),P.innerHTML=Cl,yt=n(),mt=i("hr"),Tt=n(),M(O.$$.fragment),ut=n(),D=i("p"),D.innerHTML=Bl,Jt=n(),M(K.$$.fragment),Ut=n(),jt=i("hr"),ft=n(),M(ee.$$.fragment),gt=n(),te=i("p"),te.innerHTML=vl,It=n(),M(le.$$.fragment),bt=n(),Ct=i("hr"),Bt=n(),M(se.$$.fragment),vt=n(),ne=i("p"),ne.innerHTML=Al,At=n(),M(ae.$$.fragment),Gt=n(),ie=i("p"),ie.innerHTML=Gl,_t=n(),U=i("blockquote"),U.innerHTML=_l,Zt=n(),Rt=i("hr"),$t=n(),M(oe.$$.fragment),kt=n(),re=i("p"),re.innerHTML=Zl,Et=n(),M(Me.$$.fragment),Wt=n(),M(pe.$$.fragment),Qt=n(),ce=i("p"),ce.textContent=Rl,St=n(),we=i("ul"),we.innerHTML=$l,Ht=n(),de=i("p"),de.textContent=kl,Nt=n(),Vt=i("hr"),Xt=n(),M(he.$$.fragment),Ft=n(),ye=i("p"),ye.innerHTML=El,zt=n(),M(me.$$.fragment),Yt=n(),Te=i("p"),Te.innerHTML=Wl,xt=n(),Lt=i("hr"),qt=n(),M(ue.$$.fragment),Pt=n(),Je=i("p"),Je.innerHTML=Ql,Ot=n(),M(Ue.$$.fragment),Dt=n(),je=i("p"),je.innerHTML=Sl,Kt=n(),fe=i("ul"),fe.innerHTML=Hl,el=n(),j=i("blockquote"),j.innerHTML=Nl,tl=n(),ll=i("hr"),sl=n(),M(ge.$$.fragment),nl=n(),Ie=i("ul"),Ie.innerHTML=Vl,al=n(),M(be.$$.fragment),il=n(),ve=i("p"),this.h()},l(e){const t=ql("svelte-u9bgzb",document.head);T=o(t,"META",{name:!0,content:!0}),t.forEach(l),Ae=a(e),Be=o(e,"P",{}),Xl(Be).forEach(l),Ge=a(e),p(f.$$.fragment,e),_e=a(e),p(g.$$.fragment,e),Ze=a(e),I=o(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-1rx6043"&&(I.innerHTML=Ml),Re=a(e),b=o(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-1uxmthr"&&(b.textContent=pl),$e=a(e),C=o(e,"OL",{"data-svelte-h":!0}),r(C)!=="svelte-13bszu9"&&(C.innerHTML=cl),ke=a(e),B=o(e,"P",{"data-svelte-h":!0}),r(B)!=="svelte-1mhtlb3"&&(B.textContent=wl),Ee=a(e),p(v.$$.fragment,e),We=a(e),A=o(e,"P",{"data-svelte-h":!0}),r(A)!=="svelte-7yzqig"&&(A.innerHTML=dl),Qe=a(e),p(G.$$.fragment,e),Se=a(e),_=o(e,"UL",{"data-svelte-h":!0}),r(_)!=="svelte-1bruxy"&&(_.innerHTML=hl),He=a(e),u=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(u)!=="svelte-14lgbn4"&&(u.innerHTML=yl),Ne=a(e),Ve=o(e,"HR",{}),Xe=a(e),p(Z.$$.fragment,e),Fe=a(e),R=o(e,"P",{"data-svelte-h":!0}),r(R)!=="svelte-75jqw0"&&(R.innerHTML=ml),ze=a(e),$=o(e,"P",{"data-svelte-h":!0}),r($)!=="svelte-14fvg4q"&&($.innerHTML=Tl),Ye=a(e),p(k.$$.fragment,e),xe=a(e),Le=o(e,"HR",{}),qe=a(e),p(E.$$.fragment,e),Pe=a(e),W=o(e,"P",{"data-svelte-h":!0}),r(W)!=="svelte-4x4ai4"&&(W.textContent=ul),Oe=a(e),p(Q.$$.fragment,e),De=a(e),Ke=o(e,"HR",{}),et=a(e),p(S.$$.fragment,e),tt=a(e),H=o(e,"P",{"data-svelte-h":!0}),r(H)!=="svelte-wvcamk"&&(H.innerHTML=Jl),lt=a(e),p(N.$$.fragment,e),st=a(e),nt=o(e,"HR",{}),at=a(e),p(V.$$.fragment,e),it=a(e),X=o(e,"P",{"data-svelte-h":!0}),r(X)!=="svelte-8f750j"&&(X.innerHTML=Ul),ot=a(e),F=o(e,"UL",{"data-svelte-h":!0}),r(F)!=="svelte-ciiorp"&&(F.innerHTML=jl),rt=a(e),z=o(e,"P",{"data-svelte-h":!0}),r(z)!=="svelte-vmartn"&&(z.innerHTML=fl),Mt=a(e),p(Y.$$.fragment,e),pt=a(e),J=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(J)!=="svelte-9o8n9h"&&(J.innerHTML=gl),ct=a(e),p(x.$$.fragment,e),wt=a(e),L=o(e,"P",{"data-svelte-h":!0}),r(L)!=="svelte-avdnqv"&&(L.innerHTML=Il),dt=a(e),q=o(e,"OL",{"data-svelte-h":!0}),r(q)!=="svelte-j2n64x"&&(q.innerHTML=bl),ht=a(e),P=o(e,"P",{"data-svelte-h":!0}),r(P)!=="svelte-he3dnr"&&(P.innerHTML=Cl),yt=a(e),mt=o(e,"HR",{}),Tt=a(e),p(O.$$.fragment,e),ut=a(e),D=o(e,"P",{"data-svelte-h":!0}),r(D)!=="svelte-19e32w5"&&(D.innerHTML=Bl),Jt=a(e),p(K.$$.fragment,e),Ut=a(e),jt=o(e,"HR",{}),ft=a(e),p(ee.$$.fragment,e),gt=a(e),te=o(e,"P",{"data-svelte-h":!0}),r(te)!=="svelte-1fdv46z"&&(te.innerHTML=vl),It=a(e),p(le.$$.fragment,e),bt=a(e),Ct=o(e,"HR",{}),Bt=a(e),p(se.$$.fragment,e),vt=a(e),ne=o(e,"P",{"data-svelte-h":!0}),r(ne)!=="svelte-1rn932w"&&(ne.innerHTML=Al),At=a(e),p(ae.$$.fragment,e),Gt=a(e),ie=o(e,"P",{"data-svelte-h":!0}),r(ie)!=="svelte-5xjlk7"&&(ie.innerHTML=Gl),_t=a(e),U=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(U)!=="svelte-1badvij"&&(U.innerHTML=_l),Zt=a(e),Rt=o(e,"HR",{}),$t=a(e),p(oe.$$.fragment,e),kt=a(e),re=o(e,"P",{"data-svelte-h":!0}),r(re)!=="svelte-1noo261"&&(re.innerHTML=Zl),Et=a(e),p(Me.$$.fragment,e),Wt=a(e),p(pe.$$.fragment,e),Qt=a(e),ce=o(e,"P",{"data-svelte-h":!0}),r(ce)!=="svelte-1s1rzg5"&&(ce.textContent=Rl),St=a(e),we=o(e,"UL",{"data-svelte-h":!0}),r(we)!=="svelte-jbwhci"&&(we.innerHTML=$l),Ht=a(e),de=o(e,"P",{"data-svelte-h":!0}),r(de)!=="svelte-5dsv0h"&&(de.textContent=kl),Nt=a(e),Vt=o(e,"HR",{}),Xt=a(e),p(he.$$.fragment,e),Ft=a(e),ye=o(e,"P",{"data-svelte-h":!0}),r(ye)!=="svelte-958g1w"&&(ye.innerHTML=El),zt=a(e),p(me.$$.fragment,e),Yt=a(e),Te=o(e,"P",{"data-svelte-h":!0}),r(Te)!=="svelte-gazgx1"&&(Te.innerHTML=Wl),xt=a(e),Lt=o(e,"HR",{}),qt=a(e),p(ue.$$.fragment,e),Pt=a(e),Je=o(e,"P",{"data-svelte-h":!0}),r(Je)!=="svelte-19kpoxj"&&(Je.innerHTML=Ql),Ot=a(e),p(Ue.$$.fragment,e),Dt=a(e),je=o(e,"P",{"data-svelte-h":!0}),r(je)!=="svelte-en44kf"&&(je.innerHTML=Sl),Kt=a(e),fe=o(e,"UL",{"data-svelte-h":!0}),r(fe)!=="svelte-10kwndd"&&(fe.innerHTML=Hl),el=a(e),j=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(j)!=="svelte-hpqyh3"&&(j.innerHTML=Nl),tl=a(e),ll=o(e,"HR",{}),sl=a(e),p(ge.$$.fragment,e),nl=a(e),Ie=o(e,"UL",{"data-svelte-h":!0}),r(Ie)!=="svelte-cdmijk"&&(Ie.innerHTML=Vl),al=a(e),p(be.$$.fragment,e),il=a(e),ve=o(e,"P",{}),Xl(ve).forEach(l),this.h()},h(){Ce(T,"name","hf:doc:metadata"),Ce(T,"content",es),Ce(u,"class","note"),Ce(J,"class","note"),Ce(U,"class","note"),Ce(j,"class","note")},m(e,t){Pl(document.head,T),s(e,Ae,t),s(e,Be,t),s(e,Ge,t),c(f,e,t),s(e,_e,t),c(g,e,t),s(e,Ze,t),s(e,I,t),s(e,Re,t),s(e,b,t),s(e,$e,t),s(e,C,t),s(e,ke,t),s(e,B,t),s(e,Ee,t),c(v,e,t),s(e,We,t),s(e,A,t),s(e,Qe,t),c(G,e,t),s(e,Se,t),s(e,_,t),s(e,He,t),s(e,u,t),s(e,Ne,t),s(e,Ve,t),s(e,Xe,t),c(Z,e,t),s(e,Fe,t),s(e,R,t),s(e,ze,t),s(e,$,t),s(e,Ye,t),c(k,e,t),s(e,xe,t),s(e,Le,t),s(e,qe,t),c(E,e,t),s(e,Pe,t),s(e,W,t),s(e,Oe,t),c(Q,e,t),s(e,De,t),s(e,Ke,t),s(e,et,t),c(S,e,t),s(e,tt,t),s(e,H,t),s(e,lt,t),c(N,e,t),s(e,st,t),s(e,nt,t),s(e,at,t),c(V,e,t),s(e,it,t),s(e,X,t),s(e,ot,t),s(e,F,t),s(e,rt,t),s(e,z,t),s(e,Mt,t),c(Y,e,t),s(e,pt,t),s(e,J,t),s(e,ct,t),c(x,e,t),s(e,wt,t),s(e,L,t),s(e,dt,t),s(e,q,t),s(e,ht,t),s(e,P,t),s(e,yt,t),s(e,mt,t),s(e,Tt,t),c(O,e,t),s(e,ut,t),s(e,D,t),s(e,Jt,t),c(K,e,t),s(e,Ut,t),s(e,jt,t),s(e,ft,t),c(ee,e,t),s(e,gt,t),s(e,te,t),s(e,It,t),c(le,e,t),s(e,bt,t),s(e,Ct,t),s(e,Bt,t),c(se,e,t),s(e,vt,t),s(e,ne,t),s(e,At,t),c(ae,e,t),s(e,Gt,t),s(e,ie,t),s(e,_t,t),s(e,U,t),s(e,Zt,t),s(e,Rt,t),s(e,$t,t),c(oe,e,t),s(e,kt,t),s(e,re,t),s(e,Et,t),c(Me,e,t),s(e,Wt,t),c(pe,e,t),s(e,Qt,t),s(e,ce,t),s(e,St,t),s(e,we,t),s(e,Ht,t),s(e,de,t),s(e,Nt,t),s(e,Vt,t),s(e,Xt,t),c(he,e,t),s(e,Ft,t),s(e,ye,t),s(e,zt,t),c(me,e,t),s(e,Yt,t),s(e,Te,t),s(e,xt,t),s(e,Lt,t),s(e,qt,t),c(ue,e,t),s(e,Pt,t),s(e,Je,t),s(e,Ot,t),c(Ue,e,t),s(e,Dt,t),s(e,je,t),s(e,Kt,t),s(e,fe,t),s(e,el,t),s(e,j,t),s(e,tl,t),s(e,ll,t),s(e,sl,t),c(ge,e,t),s(e,nl,t),s(e,Ie,t),s(e,al,t),c(be,e,t),s(e,il,t),s(e,ve,t),ol=!0},p:zl,i(e){ol||(w(f.$$.fragment,e),w(g.$$.fragment,e),w(v.$$.fragment,e),w(G.$$.fragment,e),w(Z.$$.fragment,e),w(k.$$.fragment,e),w(E.$$.fragment,e),w(Q.$$.fragment,e),w(S.$$.fragment,e),w(N.$$.fragment,e),w(V.$$.fragment,e),w(Y.$$.fragment,e),w(x.$$.fragment,e),w(O.$$.fragment,e),w(K.$$.fragment,e),w(ee.$$.fragment,e),w(le.$$.fragment,e),w(se.$$.fragment,e),w(ae.$$.fragment,e),w(oe.$$.fragment,e),w(Me.$$.fragment,e),w(pe.$$.fragment,e),w(he.$$.fragment,e),w(me.$$.fragment,e),w(ue.$$.fragment,e),w(Ue.$$.fragment,e),w(ge.$$.fragment,e),w(be.$$.fragment,e),ol=!0)},o(e){d(f.$$.fragment,e),d(g.$$.fragment,e),d(v.$$.fragment,e),d(G.$$.fragment,e),d(Z.$$.fragment,e),d(k.$$.fragment,e),d(E.$$.fragment,e),d(Q.$$.fragment,e),d(S.$$.fragment,e),d(N.$$.fragment,e),d(V.$$.fragment,e),d(Y.$$.fragment,e),d(x.$$.fragment,e),d(O.$$.fragment,e),d(K.$$.fragment,e),d(ee.$$.fragment,e),d(le.$$.fragment,e),d(se.$$.fragment,e),d(ae.$$.fragment,e),d(oe.$$.fragment,e),d(Me.$$.fragment,e),d(pe.$$.fragment,e),d(he.$$.fragment,e),d(me.$$.fragment,e),d(ue.$$.fragment,e),d(Ue.$$.fragment,e),d(ge.$$.fragment,e),d(be.$$.fragment,e),ol=!1},d(e){e&&(l(Ae),l(Be),l(Ge),l(_e),l(Ze),l(I),l(Re),l(b),l($e),l(C),l(ke),l(B),l(Ee),l(We),l(A),l(Qe),l(Se),l(_),l(He),l(u),l(Ne),l(Ve),l(Xe),l(Fe),l(R),l(ze),l($),l(Ye),l(xe),l(Le),l(qe),l(Pe),l(W),l(Oe),l(De),l(Ke),l(et),l(tt),l(H),l(lt),l(st),l(nt),l(at),l(it),l(X),l(ot),l(F),l(rt),l(z),l(Mt),l(pt),l(J),l(ct),l(wt),l(L),l(dt),l(q),l(ht),l(P),l(yt),l(mt),l(Tt),l(ut),l(D),l(Jt),l(Ut),l(jt),l(ft),l(gt),l(te),l(It),l(bt),l(Ct),l(Bt),l(vt),l(ne),l(At),l(Gt),l(ie),l(_t),l(U),l(Zt),l(Rt),l($t),l(kt),l(re),l(Et),l(Wt),l(Qt),l(ce),l(St),l(we),l(Ht),l(de),l(Nt),l(Vt),l(Xt),l(Ft),l(ye),l(zt),l(Yt),l(Te),l(xt),l(Lt),l(qt),l(Pt),l(Je),l(Ot),l(Dt),l(je),l(Kt),l(fe),l(el),l(j),l(tl),l(ll),l(sl),l(nl),l(Ie),l(al),l(il),l(ve)),l(T),h(f,e),h(g,e),h(v,e),h(G,e),h(Z,e),h(k,e),h(E,e),h(Q,e),h(S,e),h(N,e),h(V,e),h(Y,e),h(x,e),h(O,e),h(K,e),h(ee,e),h(le,e),h(se,e),h(ae,e),h(oe,e),h(Me,e),h(pe,e),h(he,e),h(me,e),h(ue,e),h(Ue,e),h(ge,e),h(be,e)}}}const es='{"title":"End-to-end OpenEnv walkthrough: train a reasoning agent with GRPO","local":"end-to-end-openenv-walkthrough-train-a-reasoning-agent-with-grpo","sections":[{"title":"Why this shape","local":"why-this-shape","sections":[],"depth":2},{"title":"What you’ll use","local":"what-youll-use","sections":[],"depth":2},{"title":"1. Install dependencies","local":"1-install-dependencies","sections":[],"depth":2},{"title":"2. Log in to Hugging Face","local":"2-log-in-to-hugging-face","sections":[],"depth":2},{"title":"3. Define the system prompt","local":"3-define-the-system-prompt","sections":[],"depth":2},{"title":"4. Define the environment class","local":"4-define-the-environment-class","sections":[{"title":"What the trainer does with this class","local":"what-the-trainer-does-with-this-class","sections":[],"depth":3}],"depth":2},{"title":"5. Define the reward function","local":"5-define-the-reward-function","sections":[],"depth":2},{"title":"6. Create the dataset","local":"6-create-the-dataset","sections":[],"depth":2},{"title":"7. Set the GRPO config","local":"7-set-the-grpo-config","sections":[],"depth":2},{"title":"8. Create the GRPOTrainer and start training","local":"8-create-the-grpotrainer-and-start-training","sections":[{"title":"Reading the trackio dashboard while it runs","local":"reading-the-trackio-dashboard-while-it-runs","sections":[],"depth":3}],"depth":2},{"title":"9. Publish the trained model to the Hub","local":"9-publish-the-trained-model-to-the-hub","sections":[],"depth":2},{"title":"10. Read the training reward delta","local":"10-read-the-training-reward-delta","sections":[],"depth":2},{"title":"11. Where to go next","local":"11-where-to-go-next","sections":[],"depth":2}],"depth":1}';function ts(rl){return Yl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class is extends xl{constructor(T){super(),Ll(this,T,ts,Kl,Fl,{})}}export{is as component};