Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / openenv /pr_749 /en /tutorials /end-to-end-walkthrough.html

HuggingFaceDocBuilder

about 1 month ago

download

raw

65.1 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"End-to-end OpenEnv walkthrough: train a reasoning agent with GRPO","local":"end-to-end-openenv-walkthrough-train-a-reasoning-agent-with-grpo","sections":[{"title":"Why this shape","local":"why-this-shape","sections":[],"depth":2},{"title":"What you’ll use","local":"what-youll-use","sections":[],"depth":2},{"title":"1. Install dependencies","local":"1-install-dependencies","sections":[],"depth":2},{"title":"2. Log in to Hugging Face","local":"2-log-in-to-hugging-face","sections":[],"depth":2},{"title":"3. Define the system prompt","local":"3-define-the-system-prompt","sections":[],"depth":2},{"title":"4. Define the environment class","local":"4-define-the-environment-class","sections":[{"title":"What the trainer does with this class","local":"what-the-trainer-does-with-this-class","sections":[],"depth":3}],"depth":2},{"title":"5. Define the reward function","local":"5-define-the-reward-function","sections":[],"depth":2},{"title":"6. Create the dataset","local":"6-create-the-dataset","sections":[],"depth":2},{"title":"7. Set the GRPO config","local":"7-set-the-grpo-config","sections":[],"depth":2},{"title":"8. Create the GRPOTrainer and start training","local":"8-create-the-grpotrainer-and-start-training","sections":[{"title":"Reading the trackio dashboard while it runs","local":"reading-the-trackio-dashboard-while-it-runs","sections":[],"depth":3}],"depth":2},{"title":"9. Publish the trained model to the Hub","local":"9-publish-the-trained-model-to-the-hub","sections":[],"depth":2},{"title":"10. Read the training reward delta","local":"10-read-the-training-reward-delta","sections":[],"depth":2},{"title":"11. Where to go next","local":"11-where-to-go-next","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/openenv/pr_749/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/entry/start.85477f45.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/chunks/scheduler.2b22cead.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/chunks/singletons.63566282.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/chunks/paths.dd876c7b.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/entry/app.51835dc5.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/chunks/preload-helper.0820fbc7.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/chunks/index.1a0e8013.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/nodes/0.167255c0.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/nodes/58.8cee58ce.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/chunks/Heading.c0d3f116.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.21bcf336.js">
	<link rel="modulepreload" href="/docs/openenv/pr_749/en/_app/immutable/chunks/CodeBlock.c8d73295.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"End-to-end OpenEnv walkthrough: train a reasoning agent with GRPO","local":"end-to-end-openenv-walkthrough-train-a-reasoning-agent-with-grpo","sections":[{"title":"Why this shape","local":"why-this-shape","sections":[],"depth":2},{"title":"What you’ll use","local":"what-youll-use","sections":[],"depth":2},{"title":"1. Install dependencies","local":"1-install-dependencies","sections":[],"depth":2},{"title":"2. Log in to Hugging Face","local":"2-log-in-to-hugging-face","sections":[],"depth":2},{"title":"3. Define the system prompt","local":"3-define-the-system-prompt","sections":[],"depth":2},{"title":"4. Define the environment class","local":"4-define-the-environment-class","sections":[{"title":"What the trainer does with this class","local":"what-the-trainer-does-with-this-class","sections":[],"depth":3}],"depth":2},{"title":"5. Define the reward function","local":"5-define-the-reward-function","sections":[],"depth":2},{"title":"6. Create the dataset","local":"6-create-the-dataset","sections":[],"depth":2},{"title":"7. Set the GRPO config","local":"7-set-the-grpo-config","sections":[],"depth":2},{"title":"8. Create the GRPOTrainer and start training","local":"8-create-the-grpotrainer-and-start-training","sections":[{"title":"Reading the trackio dashboard while it runs","local":"reading-the-trackio-dashboard-while-it-runs","sections":[],"depth":3}],"depth":2},{"title":"9. Publish the trained model to the Hub","local":"9-publish-the-trained-model-to-the-hub","sections":[],"depth":2},{"title":"10. Read the training reward delta","local":"10-read-the-training-reward-delta","sections":[],"depth":2},{"title":"11. Where to go next","local":"11-where-to-go-next","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="end-to-end-openenv-walkthrough-train-a-reasoning-agent-with-grpo" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#end-to-end-openenv-walkthrough-train-a-reasoning-agent-with-grpo"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>End-to-end OpenEnv walkthrough: train a reasoning agent with GRPO</span></h1> <p data-svelte-h="svelte-1rx6043"><a href="https://colab.research.google.com/github/huggingface/OpenEnv/blob/main/examples/end_to_end_walkthrough.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a></p> <p data-svelte-h="svelte-1uxmthr">In this tutorial you’ll take a small open-weight model, an OpenEnv environment, and TRL, and run the full training pipeline end-to-end:</p> <ol data-svelte-h="svelte-13bszu9"><li>Connect to a hosted environment.</li> <li>Wire it into TRL via the <code>environment_factory</code> pattern.</li> <li>Fine-tune with <strong>GRPO</strong> (Group Relative Policy Optimization).</li> <li>Read the reward delta from the training logs to see how much the policy improved.</li> <li>Publish the trained model to the Hub.</li></ol> <p data-svelte-h="svelte-1mhtlb3">The goal is to see the whole pipeline as one coherent narrative — model, environment, training, metric — rather than three separate articles.</p> <h2 class="relative group"><a id="why-this-shape" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#why-this-shape"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Why this shape</span></h2> <p data-svelte-h="svelte-7yzqig">We pair <strong>GRPO</strong> with a <strong>procedural</strong> task on purpose. GRPO is a value-free RL method that ranks several rollouts of the same prompt against each other, so the only signal it needs is a per-rollout scalar reward — exactly what an environment can return after a <code>step</code>. Procedural means the env generates a fresh question every episode rather than serving a fixed dataset, so the model has to <em>generalize</em> over the family of problems instead of memorizing specific items.</p> <h2 class="relative group"><a id="what-youll-use" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#what-youll-use"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>What you’ll use</span></h2> <ul data-svelte-h="svelte-1bruxy"><li><strong>Model</strong>: <a href="https://huggingface.co/Qwen/Qwen3-1.7B" rel="nofollow"><code>Qwen/Qwen3-1.7B</code></a> — fits a single A100 (40 GB) at the settings below and is large enough for GRPO to move the needle. For smaller GPUs, swap to <a href="https://huggingface.co/Qwen/Qwen3-0.6B" rel="nofollow"><code>Qwen/Qwen3-0.6B</code></a>.</li> <li><strong>Environment</strong>: <a href="https://github.com/huggingface/OpenEnv/tree/main/envs/reasoning_gym_env" rel="nofollow"><code>reasoning_gym_env</code></a>, an OpenEnv wrapper around the <a href="https://github.com/open-thought/reasoning-gym" rel="nofollow">Reasoning Gym</a> library. Each episode is a single Q→A.</li> <li><strong>Dataset</strong>: <code>chain_sum</code> from Reasoning Gym — chains of integer additions like <code>Compute 17 + 4 + 22 + 9</code>. Procedurally generated, so every rollout sees a fresh problem.</li> <li><strong>Trainer</strong>: <a href="https://huggingface.co/docs/trl/main/en/grpo_trainer" rel="nofollow">TRL <code>GRPOTrainer</code></a> with <code>environment_factory</code>.</li></ul> <blockquote class="note" data-svelte-h="svelte-14lgbn4"><p>This tutorial runs through training; on a single A100 (40 GB) the recipe completes in roughly an hour at the suggested settings, peaking around ~38 GB of VRAM. T4 (16 GB) won’t fit Qwen3-1.7B at these settings — see the model bullet above for the smaller-GPU swap. The exact reward numbers you see will vary with seed and budget — the point is to watch the reward curve climb and report the delta.</p></blockquote> <hr> <h2 class="relative group"><a id="1-install-dependencies" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#1-install-dependencies"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>1. Install dependencies</span></h2> <p data-svelte-h="svelte-75jqw0">This tutorial connects to <a href="https://huggingface.co/spaces/sergiopaniego/reasoning_gym" rel="nofollow"><code>sergiopaniego/reasoning_gym</code></a>. For your own training runs, deploy your own copy first by running <code>openenv push --repo-id <your-username>/reasoning_gym</code> inside <code>envs/reasoning_gym_env/</code> of the OpenEnv repo, then replace <code>sergiopaniego</code> with your username in the install line and the <code>base_url=</code> strings further down.</p> <p data-svelte-h="svelte-14fvg4q">Install pip dependencies — keep them as separate cells (don’t merge into one <code>pip install</code>):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START -->!pip install -q trl
	!pip install -q openenv
	!pip install -q --no-deps git+https://huggingface.co/spaces/sergiopaniego/reasoning_gym
	!pip install -Uq <span class="hljs-string">"transformers>=5.3.0"</span> <span class="hljs-comment"># 5.3+ has the `environment_factory` integration TRL needs</span>
	!pip install -q trackio jmespath<!-- HTML_TAG_END --></pre></div> <hr> <h2 class="relative group"><a id="2-log-in-to-hugging-face" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#2-log-in-to-hugging-face"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>2. Log in to Hugging Face</span></h2> <p data-svelte-h="svelte-4x4ai4">You’ll need to be logged in to download the base model and (optionally) push the trained checkpoint.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

	notebook_login()<!-- HTML_TAG_END --></pre></div> <hr> <h2 class="relative group"><a id="3-define-the-system-prompt" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#3-define-the-system-prompt"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>3. Define the system prompt</span></h2> <p data-svelte-h="svelte-wvcamk">The model will be asked to use a single tool, <code>answer</code>, to submit its final number. The prompt makes that explicit.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START -->prompt = <span class="hljs-string">"""You are a careful arithmetic assistant.

	You will be given a chain of integer additions. Compute the result and submit it as a single number.

	Rules:
	1. Read the question carefully.
	2. Use the tool `answer` exactly once with your final number.
	3. The answer must be a single integer with no units or explanation.
	"""</span><!-- HTML_TAG_END --></pre></div> <hr> <h2 class="relative group"><a id="4-define-the-environment-class" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#4-define-the-environment-class"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>4. Define the environment class</span></h2> <p data-svelte-h="svelte-8f750j">The <code>environment_factory</code> pattern asks for a Python class that the trainer can instantiate per rollout. It needs:</p> <ul data-svelte-h="svelte-ciiorp"><li>An <code>__init__</code> that opens a connection to the underlying environment.</li> <li>A <code>reset(**kwargs)</code> method that starts a new episode and returns the initial observation as a string (the question, in our case).</li> <li>One or more <em>tool methods</em> — public methods with docstrings — that the trainer auto-discovers and exposes as tools to the model. Each call corresponds to one <code>env.step</code> on the underlying environment.</li></ul> <p data-svelte-h="svelte-vmartn">Because Reasoning Gym episodes are <strong>single-step</strong> (one question → one answer → done), the wrapper is small.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> random

	<span class="hljs-keyword">from</span> reasoning_gym_env <span class="hljs-keyword">import</span> ReasoningGymAction, ReasoningGymEnv


	<span class="hljs-keyword">class</span> <span class="hljs-title class_">ReasoningGymTrainEnv</span>:
	<span class="hljs-string">"""Environment wrapper for GRPO training on chain_sum.

	Each rollout episode = one question → one `answer` tool call → done.
	"""</span>

	DATASET_NAME = <span class="hljs-string">"chain_sum"</span>
	DATASET_SIZE = <span class="hljs-number">1000</span>
	DATASET_CONFIG = {
	<span class="hljs-string">"min_terms"</span>: <span class="hljs-number">2</span>,
	<span class="hljs-string">"max_terms"</span>: <span class="hljs-number">3</span>,
	<span class="hljs-string">"min_digits"</span>: <span class="hljs-number">2</span>,
	<span class="hljs-string">"max_digits"</span>: <span class="hljs-number">2</span>,
	}

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
	<span class="hljs-comment"># `EnvClient` subclasses are async by default; `.sync()` returns a</span>
	<span class="hljs-comment"># synchronous wrapper so the trainer can call our tool methods directly.</span>
	self.client = ReasoningGymEnv(base_url=<span class="hljs-string">"https://sergiopaniego-reasoning-gym.hf.space"</span>).sync()
	<span class="hljs-comment"># Random seed per instance so the parallel envs the trainer creates</span>
	<span class="hljs-comment"># don't all iterate over the same question sequence.</span>
	self._dataset_seed = random.randint(<span class="hljs-number">0</span>, <span class="hljs-number">2</span>**<span class="hljs-number">31</span> - <span class="hljs-number">1</span>)
	self._initialized = <span class="hljs-literal">False</span>
	self.reward = <span class="hljs-number">0.0</span>
	self.done = <span class="hljs-literal">False</span>

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, **kwargs</span>) -> <span class="hljs-built_in">str</span>:
	<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> self._initialized:
	<span class="hljs-comment"># First reset: configure the dataset (name + config + seed + size).</span>
	result = self.client.reset(
	dataset_name=self.DATASET_NAME,
	dataset_config=self.DATASET_CONFIG,
	seed=self._dataset_seed,
	size=self.DATASET_SIZE,
	)
	self._initialized = <span class="hljs-literal">True</span>
	<span class="hljs-keyword">else</span>:
	<span class="hljs-comment"># Subsequent resets: no args → server returns the next question</span>
	<span class="hljs-comment"># from the same dataset iterator. Re-sending the config would</span>
	<span class="hljs-comment"># rebuild the dataset and rewind to question 0.</span>
	result = self.client.reset()
	self.reward = <span class="hljs-number">0.0</span>
	self.done = <span class="hljs-literal">False</span>
	<span class="hljs-keyword">return</span> result.observation.question

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">answer</span>(<span class="hljs-params">self, answer: <span class="hljs-built_in">str</span></span>) -> <span class="hljs-built_in">str</span>:
	<span class="hljs-string">"""Submit the final answer for the current question.

	Args:
	answer: The agent's answer (will be parsed as a number server-side).

	Returns:
	A short feedback string with the score and the correct answer.
	"""</span>
	<span class="hljs-keyword">if</span> self.done:
	<span class="hljs-keyword">raise</span> ValueError(<span class="hljs-string">"Episode is already finished."</span>)
	<span class="hljs-comment"># The model often emits `answer` as a JSON int (e.g. `7`) even though</span>
	<span class="hljs-comment"># the tool schema declares string — coerce so pydantic validation on</span>
	<span class="hljs-comment"># `ReasoningGymAction` doesn't reject the rollout.</span>
	result = self.client.step(ReasoningGymAction(answer=<span class="hljs-built_in">str</span>(answer)))
	self.reward = <span class="hljs-built_in">float</span>(result.observation.score <span class="hljs-keyword">or</span> <span class="hljs-number">0.0</span>)
	self.done = <span class="hljs-literal">True</span>
	<span class="hljs-keyword">return</span> <span class="hljs-string">f"score=<span class="hljs-subst">{self.reward}</span> correct=<span class="hljs-subst">{result.observation.correct_answer}</span>"</span><!-- HTML_TAG_END --></pre></div> <blockquote class="note" data-svelte-h="svelte-9o8n9h"><p>Replace the <code>base_url</code> with your own deployment if you’ve pushed <code>reasoning_gym_env</code> to your own Space — the hosted versions have limited concurrency and are intended for tutorials and small experiments.</p></blockquote> <h3 class="relative group"><a id="what-the-trainer-does-with-this-class" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#what-the-trainer-does-with-this-class"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>What the trainer does with this class</span></h3> <p data-svelte-h="svelte-avdnqv">It helps to picture the runtime loop. At init the trainer creates <code>gradient_accumulation_steps × per_device_train_batch_size</code> instances of <code>ReasoningGymTrainEnv</code> — these stay alive across optimizer steps. Per generation batch it then does, <strong>for each instance in parallel</strong>:</p> <ol data-svelte-h="svelte-j2n64x"><li><code>env.reset(row)</code> — opens (or reuses) the WebSocket session and returns the question string.</li> <li>The model is conditioned on that question, generates <code>num_generations</code> candidate completions, and the trainer parses any <code><tool_call></code> blocks out of each.</li> <li>For each parsed call it dispatches to the matching tool method (here, <code>answer(...)</code>) and feeds the return value back to the model as a <code><tool_response></code>.</li> <li>When the env signals <code>done=True</code>, the rollout ends and the trainer reads <code>env.reward</code>.</li> <li>GRPO computes one advantage per completion (relative to the group’s mean reward), updates the policy, and the cycle repeats.</li></ol> <p data-svelte-h="svelte-he3dnr">That’s why the wrapper only needs three things — a connection in <code>__init__</code>, a <code>reset</code> that returns the initial obs, and one or more tool methods that update <code>self.reward</code>/<code>self.done</code>.</p> <hr> <h2 class="relative group"><a id="5-define-the-reward-function" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#5-define-the-reward-function"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>5. Define the reward function</span></h2> <p data-svelte-h="svelte-19e32w5">The reward function receives the list of environment instances after each rollout. Each instance already tracks its own reward (set inside <code>answer()</code>), so we just read it back.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">environments, kwargs</span>) -> <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
	<span class="hljs-keyword">return</span> [env.reward <span class="hljs-keyword">for</span> env <span class="hljs-keyword">in</span> environments]<!-- HTML_TAG_END --></pre></div> <hr> <h2 class="relative group"><a id="6-create-the-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#6-create-the-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>6. Create the dataset</span></h2> <p data-svelte-h="svelte-1fdv46z">Each row in the training dataset triggers one rollout episode. The prompt is identical across rows because the <em>environment</em> supplies the per-episode question — we’re using the dataset purely to control how many episodes the trainer runs.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

	dataset = Dataset.from_dict(
	{<span class="hljs-string">"prompt"</span>: [[{<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: prompt}] <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>)]}
	)<!-- HTML_TAG_END --></pre></div> <hr> <h2 class="relative group"><a id="7-set-the-grpo-config" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#7-set-the-grpo-config"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>7. Set the GRPO config</span></h2> <p data-svelte-h="svelte-1rn932w">These settings mirror the <a href="wordle-grpo">Wordle GRPO tutorial</a> and are tuned for a single A100 (40 GB). Bigger GPUs can raise <code>per_device_train_batch_size</code> and <code>num_generations</code>; smaller GPUs should drop to Qwen3-0.6B and shrink <code>max_completion_length</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig

	output_dir = <span class="hljs-string">"reasoning-gym-chain-sum-Qwen3-1.7B"</span>

	grpo_config = GRPOConfig(
	num_train_epochs=<span class="hljs-number">1</span>,
	max_steps=<span class="hljs-number">150</span>,
	learning_rate=<span class="hljs-number">1e-6</span>,
	gradient_accumulation_steps=<span class="hljs-number">4</span>,
	per_device_train_batch_size=<span class="hljs-number">1</span>,
	warmup_steps=<span class="hljs-number">10</span>,
	optim=<span class="hljs-string">"adamw_torch"</span>,
	max_grad_norm=<span class="hljs-number">1.0</span>,
	num_generations=<span class="hljs-number">2</span>,
	max_completion_length=<span class="hljs-number">256</span>,
	log_completions=<span class="hljs-literal">True</span>,
	num_completions_to_print=<span class="hljs-number">2</span>,
	chat_template_kwargs={<span class="hljs-string">"enable_thinking"</span>: <span class="hljs-literal">False</span>},
	output_dir=output_dir,
	report_to=<span class="hljs-string">"trackio"</span>,
	trackio_space_id=output_dir,
	logging_steps=<span class="hljs-number">10</span>,
	gradient_checkpointing=<span class="hljs-literal">True</span>,
	save_strategy=<span class="hljs-string">"no"</span>,
	push_to_hub=<span class="hljs-literal">True</span>,
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-5xjlk7">A few of the choices above worth flagging: <code>max_steps=150</code> caps the run before saturation (see <em>Reading the dashboard</em> below). <code>gradient_accumulation_steps=4</code> keeps the parallel env count at <code>1 × 4 = 4</code>, well under the server’s default concurrency limit. <code>save_strategy="no"</code> skips intermediate checkpoints so the run stays quiet — we push the final model explicitly in section 9. <code>use_vllm</code> is left at its default (<code>False</code>); enabling it speeds up rollouts on bare-metal but its distributed init breaks under IPython.</p> <blockquote class="note" data-svelte-h="svelte-1badvij"><p><code>chat_template_kwargs={"enable_thinking": False}</code> disables Qwen3’s thinking mode so the model emits tool calls directly instead of reasoning tokens first. For a pure tool-use task like this one that’s what you want; for harder math you may benefit from re-enabling it and growing <code>max_completion_length</code>.</p></blockquote> <hr> <h2 class="relative group"><a id="8-create-the-grpotrainer-and-start-training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#8-create-the-grpotrainer-and-start-training"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>8. Create the GRPOTrainer and start training</span></h2> <p data-svelte-h="svelte-1noo261"><code>environment_factory=ReasoningGymTrainEnv</code> is the only piece wiring our wrapper into the training loop.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer

	MODEL_NAME = <span class="hljs-string">"Qwen/Qwen3-1.7B"</span>

	trainer = GRPOTrainer(
	model=MODEL_NAME,
	reward_funcs=reward_func,
	train_dataset=dataset,
	args=grpo_config,
	environment_factory=ReasoningGymTrainEnv,
	)

	trainer.train()<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="reading-the-trackio-dashboard-while-it-runs" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#reading-the-trackio-dashboard-while-it-runs"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Reading the trackio dashboard while it runs</span></h3> <p data-svelte-h="svelte-1s1rzg5">Open the Trackio Space linked in the trainer logs to follow the run live. A healthy GRPO trajectory looks roughly like this:</p> <ul data-svelte-h="svelte-jbwhci"><li><strong><code>reward</code></strong> climbs from your baseline toward <code>1.0</code> over the first ~100 steps. A flat line near 0 means the task is too hard for the base model; a flat line near 1 means it’s too easy — adjust <code>DATASET_CONFIG</code> in either case.</li> <li><strong><code>reward_std</code></strong> starts moderate and <em>drops</em> as the policy converges (most rollouts succeed). Persistent zero means every rollout in the group gives the same score → no advantage signal → no learning. Bump <code>num_generations</code> or task difficulty.</li> <li><strong><code>frac_reward_zero_std</code></strong> is the fraction of groups where every rollout has the same reward — when it climbs toward 1.0 you’ve saturated.</li> <li><strong><code>entropy</code></strong> stays low while the model is learning. Once <code>reward</code> saturates, <code>entropy</code> typically rises again because the policy gradient is zero and only the KL penalty against the reference model is active — at that point further training is net-negative. Stop with a kernel interrupt or trust <code>max_steps</code>.</li> <li><strong><code>grad_norm</code></strong> decays toward zero as gradients become uninformative; same saturation signal.</li></ul> <p data-svelte-h="svelte-5dsv0h">Once training finishes, the model in the running process has been fine-tuned in place.</p> <hr> <h2 class="relative group"><a id="9-publish-the-trained-model-to-the-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#9-publish-the-trained-model-to-the-hub"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>9. Publish the trained model to the Hub</span></h2> <p data-svelte-h="svelte-958g1w"><code>save_strategy="no"</code> means the trainer didn’t write any intermediate checkpoints. Push the final model explicitly so others can reuse it (and so the experiment is reproducible from the Hub):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START -->trainer.push_to_hub(commit_message=<span class="hljs-string">"GRPO fine-tune on reasoning_gym chain_sum"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-gazgx1">The repo is derived automatically from <code>output_dir</code> (or <code>hub_model_id</code> if set in <code>GRPOConfig</code>). After this completes, the model lives at <code>https://huggingface.co/<your-username>/reasoning-gym-chain-sum-Qwen3-1.7B</code> and anyone can load it with <code>AutoModelForCausalLM.from_pretrained(...)</code>.</p> <hr> <h2 class="relative group"><a id="10-read-the-training-reward-delta" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#10-read-the-training-reward-delta"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>10. Read the training reward delta</span></h2> <p data-svelte-h="svelte-19kpoxj">Every rollout the trainer ran left a <code>reward</code> entry in <code>trainer.state.log_history</code>. Comparing the first few logged rewards (the model’s starting capability) to the last few (after training) gives a clean before/after number — same metric, same distribution, no second eval pass required.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> statistics

	rewards = [log[<span class="hljs-string">"reward"</span>] <span class="hljs-keyword">for</span> log <span class="hljs-keyword">in</span> trainer.state.log_history <span class="hljs-keyword">if</span> <span class="hljs-string">"reward"</span> <span class="hljs-keyword">in</span> log]

	<span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(rewards) < <span class="hljs-number">5</span>:
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"Only <span class="hljs-subst">{<span class="hljs-built_in">len</span>(rewards)}</span> reward entries logged — train for a few more `logging_steps` and re-run."</span>)
	<span class="hljs-keyword">else</span>:
	initial = statistics.mean(rewards[:<span class="hljs-number">5</span>])
	final = statistics.mean(rewards[-<span class="hljs-number">5</span>:])
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"Initial reward (first 5 logs avg): <span class="hljs-subst">{initial:<span class="hljs-number">.2</span>%}</span>"</span>)
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"Final reward (last 5 logs avg): <span class="hljs-subst">{final:<span class="hljs-number">.2</span>%}</span>"</span>)
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"Delta: <span class="hljs-subst">{(final - initial) * <span class="hljs-number">100</span>:+<span class="hljs-number">.2</span>f}</span> pp"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-en44kf">A delta of <strong>+10 to +30 pp</strong> is what you should expect at this difficulty; outside that range:</p> <ul data-svelte-h="svelte-10kwndd"><li><strong>Δ ≈ 0 pp, initial already high (≥90%)</strong> — <code>DATASET_CONFIG</code> is too easy; the model already solves it before training. Bump <code>min_terms</code> / <code>min_digits</code>.</li> <li><strong>Δ ≈ 0 pp, initial very low (≤20%)</strong> — task is too hard for the base model to ever stumble onto a correct answer, so GRPO has no positive rollouts to learn from. Lower <code>min_terms</code> / <code>min_digits</code>. If the reward stays near zero even at minimum difficulty, the bottleneck is likely <strong>format compliance</strong> rather than task difficulty — the model never produces a valid <code><tool_call></code> so the env cannot score it. See the <a href="sft-warmup">SFT warm-up tutorial</a> for how to fix this before returning to GRPO.</li> <li><strong>Δ negative</strong> — you trained past saturation: once <code>reward</code> plateaus, the KL penalty starts pulling the policy back toward the reference. Reduce <code>max_steps</code> so training stops while it’s still net-improving.</li></ul> <blockquote class="note" data-svelte-h="svelte-hpqyh3"><p>This delta is measured <em>during training</em> — same prompt format, same env, same procedural distribution that produced each rollout. It’s the most direct way to ask “did the policy improve over the run?“. A more rigorous protocol — generating completions on a held-out split with a separate evaluation harness — is what frameworks like <a href="https://inspect.aisi.org.uk/" rel="nofollow">Inspect AI</a> are designed for; that’s a follow-up rather than part of this walkthrough.</p></blockquote> <hr> <h2 class="relative group"><a id="11-where-to-go-next" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#11-where-to-go-next"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>11. Where to go next</span></h2> <ul data-svelte-h="svelte-cdmijk"><li><strong>Swap the dataset.</strong> <code>chain_sum</code> is one of ~100 datasets in <a href="https://github.com/open-thought/reasoning-gym" rel="nofollow">Reasoning Gym</a> — try <code>simple_equations</code>, <code>letter_counting</code>, or <code>propositional_logic</code> by changing <code>DATASET_NAME</code> and re-running the same recipe.</li> <li><strong>Try a different environment.</strong> The same <code>environment_factory</code> shape works for any OpenEnv environment with a small tool surface — browse the <a href="../environments">environment catalog</a> for ideas.</li> <li><strong>Use SFT as a warm-start.</strong> If format compliance is the bottleneck (initial reward near zero regardless of difficulty), the <a href="sft-warmup">SFT warm-up tutorial</a> shows how to collect teacher rollouts, filter by reward, and fine-tune a student model — so GRPO starts with non-zero <code>reward_std</code> from the first batch.</li> <li><strong>Read the other tutorials.</strong> <a href="wordle-grpo">Wordle GRPO</a> covers the multi-step variant; the full list is in the <a href="index">tutorials index</a>.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/openenv/blob/main/docs/source/tutorials/end-to-end-walkthrough.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1qwoa43 = {
	assets: "/docs/openenv/pr_749/en",
	base: "/docs/openenv/pr_749/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/openenv/pr_749/en/_app/immutable/entry/start.85477f45.js"),
	import("/docs/openenv/pr_749/en/_app/immutable/entry/app.51835dc5.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 58],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 65.1 kB
Xet hash:: f3c5446f8f9853ed2f894ad1f5bef93f6d7f773ad5e5a76df9ab9c069c6be015

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.