Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"OpenEnv Integration for Training LLMs with Environments","local":"openenv-integration-for-training-llms-with-environments","sections":[{"title":"When to use environments","local":"when-to-use-environments","sections":[],"depth":2},{"title":"Choosing an environment integration","local":"choosing-an-environment-integration","sections":[],"depth":2},{"title":"Installation","local":"installation","sections":[],"depth":2},{"title":"Quick start","local":"quick-start","sections":[],"depth":2},{"title":"How environment_factory works","local":"how-environmentfactory-works","sections":[{"title":"Environment class requirements","local":"environment-class-requirements","sections":[],"depth":3},{"title":"Tips for environment classes","local":"tips-for-environment-classes","sections":[],"depth":3},{"title":"Reward functions","local":"reward-functions","sections":[],"depth":3},{"title":"Tips for reward functions","local":"tips-for-reward-functions","sections":[],"depth":3},{"title":"max_completion_length in multi-turn episodes","local":"maxcompletionlength-in-multi-turn-episodes","sections":[],"depth":3}],"depth":2},{"title":"Advanced example: Wordle","local":"advanced-example-wordle","sections":[{"title":"The TextArena Environment","local":"the-textarena-environment","sections":[],"depth":3},{"title":"Why Wordle?","local":"why-wordle","sections":[],"depth":3},{"title":"Environment class","local":"environment-class","sections":[],"depth":3},{"title":"Reward function and training","local":"reward-function-and-training","sections":[],"depth":3},{"title":"Running the example","local":"running-the-example","sections":[],"depth":3},{"title":"Results","local":"results","sections":[],"depth":3}],"depth":2},{"title":"Multi-environment training","local":"multi-environment-training","sections":[{"title":"How it works","local":"how-it-works","sections":[],"depth":3},{"title":"Example: Wordle + Catch","local":"example-wordle--catch","sections":[],"depth":3},{"title":"Per-environment reward functions","local":"per-environment-reward-functions","sections":[],"depth":3},{"title":"Dataset with environment routing","local":"dataset-with-environment-routing","sections":[],"depth":3},{"title":"Running the multi-environment example","local":"running-the-multi-environment-example","sections":[],"depth":3}],"depth":2},{"title":"Running the environments","local":"running-the-environments","sections":[],"depth":2},{"title":"Environments catalog","local":"environments-catalog","sections":[],"depth":2},{"title":"Server concurrency","local":"server-concurrency","sections":[],"depth":2},{"title":"environment_factory vs rollout_func","local":"environmentfactory-vs-rolloutfunc","sections":[],"depth":2},{"title":"Training on harnesses: training a real coding agent (opencode)","local":"training-on-harnesses-training-a-real-coding-agent-opencode","sections":[{"title":"How it works","local":"how-it-works","sections":[],"depth":3},{"title":"Wiring","local":"wiring","sections":[],"depth":3},{"title":"What you need to define","local":"what-you-need-to-define","sections":[{"title":"rollout_reward_fn : how to score a rollout","local":"rolloutrewardfn--how-to-score-a-rollout","sections":[],"depth":4},{"title":"train_turn_fn : which turns to reinforce","local":"trainturnfn--which-turns-to-reinforce","sections":[],"depth":4},{"title":"agent_turn_fn : which trace entries are real agent turns","local":"agentturnfn--which-trace-entries-are-real-agent-turns","sections":[],"depth":4}],"depth":3},{"title":"Requirements","local":"requirements","sections":[],"depth":3}],"depth":2}],"depth":1}"/> | |
| <link href="/docs/trl/main/en/_app/immutable/entry/start.8G6YWryu.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/chunks/Cl0ox_Hs.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/chunks/DzoEXNfn.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/entry/app.B9nZyitD.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/chunks/BhQcX8Hn.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/chunks/CF2wz0qP.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/chunks/Dq4Qqmjh.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/nodes/0.DKi0M1L-.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/chunks/D7b5vxdd.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/nodes/3.8N2w9UcQ.js" rel="modulepreload"> | |
| <link href="/docs/trl/main/en/_app/immutable/chunks/DaRxB60a.js" rel="modulepreload"> | |
| <!--1t5g6vs--><meta name="hf:doc:metadata" content="{"title":"OpenEnv Integration for Training LLMs with Environments","local":"openenv-integration-for-training-llms-with-environments","sections":[{"title":"When to use environments","local":"when-to-use-environments","sections":[],"depth":2},{"title":"Choosing an environment integration","local":"choosing-an-environment-integration","sections":[],"depth":2},{"title":"Installation","local":"installation","sections":[],"depth":2},{"title":"Quick start","local":"quick-start","sections":[],"depth":2},{"title":"How environment_factory works","local":"how-environmentfactory-works","sections":[{"title":"Environment class requirements","local":"environment-class-requirements","sections":[],"depth":3},{"title":"Tips for environment classes","local":"tips-for-environment-classes","sections":[],"depth":3},{"title":"Reward functions","local":"reward-functions","sections":[],"depth":3},{"title":"Tips for reward functions","local":"tips-for-reward-functions","sections":[],"depth":3},{"title":"max_completion_length in multi-turn episodes","local":"maxcompletionlength-in-multi-turn-episodes","sections":[],"depth":3}],"depth":2},{"title":"Advanced example: Wordle","local":"advanced-example-wordle","sections":[{"title":"The TextArena Environment","local":"the-textarena-environment","sections":[],"depth":3},{"title":"Why Wordle?","local":"why-wordle","sections":[],"depth":3},{"title":"Environment class","local":"environment-class","sections":[],"depth":3},{"title":"Reward function and training","local":"reward-function-and-training","sections":[],"depth":3},{"title":"Running the example","local":"running-the-example","sections":[],"depth":3},{"title":"Results","local":"results","sections":[],"depth":3}],"depth":2},{"title":"Multi-environment training","local":"multi-environment-training","sections":[{"title":"How it works","local":"how-it-works","sections":[],"depth":3},{"title":"Example: Wordle + Catch","local":"example-wordle--catch","sections":[],"depth":3},{"title":"Per-environment reward functions","local":"per-environment-reward-functions","sections":[],"depth":3},{"title":"Dataset with environment routing","local":"dataset-with-environment-routing","sections":[],"depth":3},{"title":"Running the multi-environment example","local":"running-the-multi-environment-example","sections":[],"depth":3}],"depth":2},{"title":"Running the environments","local":"running-the-environments","sections":[],"depth":2},{"title":"Environments catalog","local":"environments-catalog","sections":[],"depth":2},{"title":"Server concurrency","local":"server-concurrency","sections":[],"depth":2},{"title":"environment_factory vs rollout_func","local":"environmentfactory-vs-rolloutfunc","sections":[],"depth":2},{"title":"Training on harnesses: training a real coding agent (opencode)","local":"training-on-harnesses-training-a-real-coding-agent-opencode","sections":[{"title":"How it works","local":"how-it-works","sections":[],"depth":3},{"title":"Wiring","local":"wiring","sections":[],"depth":3},{"title":"What you need to define","local":"what-you-need-to-define","sections":[{"title":"rollout_reward_fn : how to score a rollout","local":"rolloutrewardfn--how-to-score-a-rollout","sections":[],"depth":4},{"title":"train_turn_fn : which turns to reinforce","local":"trainturnfn--which-turns-to-reinforce","sections":[],"depth":4},{"title":"agent_turn_fn : which trace entries are real agent turns","local":"agentturnfn--which-trace-entries-are-real-agent-turns","sections":[],"depth":4}],"depth":3},{"title":"Requirements","local":"requirements","sections":[],"depth":3}],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/trl/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="openenv-integration-for-training-llms-with-environments" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#openenv-integration-for-training-llms-with-environments"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>OpenEnv Integration for Training LLMs with Environments</span></h1><!--]--><!----> <p><a href="https://github.com/huggingface/OpenEnv" rel="nofollow">OpenEnv</a> is an open-source framework for defining, deploying, and interacting with environments in reinforcement learning (RL) and agentic workflows. It provides standardized APIs for environment interaction and supports running environments as backend servers (via WebSocket or containerised execution). You can find a collection of ready-to-use OpenEnv environments on the <a href="https://huggingface.co/collections/openenv/openenv-environment-hub" rel="nofollow">Hugging Face Hub</a>.</p> <p>This guide covers <strong>how to integrate OpenEnv with TRL</strong>. For more on OpenEnv itself, see the <a href="https://huggingface.co/docs/openenv" rel="nofollow">OpenEnv docs</a>.</p> <blockquote class="note"><p>You can explore ready-to-use example <a href="example_overview#openenv-scripts">scripts</a> and <a href="example_overview#openenv-notebooks">notebooks</a> in the Examples Overview.</p></blockquote> <!--[1--><h2 class="relative group"><a id="when-to-use-environments" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#when-to-use-environments"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>When to use environments</span></h2><!--]--><!----> <p><a href="/docs/trl/main/en/gspo_token#trl.GRPOTrainer">GRPOTrainer</a> can be used to train agents. For agentic tasks, it supports two modes: <strong>tools</strong>, where the model can call external functions but each call is stateless and independent, and <strong>environments</strong>, which maintain state across turns, enabling genuine multi-turn interaction where the agent’s actions shape future observations. Use environments when continuity matters — for example, navigating a game, browsing a web page, or any task where what the agent sees next depends on what it did before.</p> <!--[1--><h2 class="relative group"><a id="choosing-an-environment-integration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#choosing-an-environment-integration"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Choosing an environment integration</span></h2><!--]--><!----> <p>OpenEnv is the native path documented here. Two further integrations — <a href="openreward">OpenReward</a> and <a href="harbor">Harbor</a> — conform to the same <code>environment_factory</code> contract and are interchangeable at the TRL level. See the <a href="grpo_trainer#agent-training">comparison of environment integrations</a> in the GRPO guide to pick the one whose ecosystem fits your task.</p> <!--[1--><h2 class="relative group"><a id="installation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#installation"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Installation</span></h2><!--]--><!----> <p>OpenEnv environments are hosted as Hugging Face Spaces, which are also pip-installable Git repositories:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># Echo environment</span> | |
| pip install <span class="hljs-string">"openenv-echo-env @ git+https://huggingface.co/spaces/openenv/echo_env"</span> | |
| <span class="hljs-comment"># Wordle (TextArena) environment</span> | |
| pip install <span class="hljs-string">"openenv-textarena @ git+https://huggingface.co/spaces/openenv/wordle"</span> | |
| <span class="hljs-comment"># Catch (OpenSpiel) environment</span> | |
| pip install <span class="hljs-string">"openenv-openspiel-env @ git+https://huggingface.co/spaces/openenv/openspiel_env"</span><!----></pre></div><!----> <p>This installs the <strong>environment client</strong> (e.g., <code>EchoEnv</code>) that communicates with the remote environment server via WebSocket, along with the action/observation models and all required dependencies (including <code>openenv</code>).</p> <blockquote class="tip"><p>You can find the install command for any environment on its HF Space page. Click the <strong>⋮ (three dots)</strong> menu and select <strong>“Use this Space”</strong> to see the install instructions.</p></blockquote> <blockquote class="tip"><p>You can also install the core package from PyPI with <code>pip install "openenv[core]>=0.3.1"</code>, but note that environment-specific dependencies may need to be installed separately.</p></blockquote> <p>For development, you can clone the OpenEnv repo and install locally:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->git <span class="hljs-built_in">clone</span> https://github.com/huggingface/OpenEnv.git | |
| <span class="hljs-built_in">cd</span> OpenEnv/envs/echo_env | |
| pip install -e .<!----></pre></div><!----> <blockquote class="note"><p>Each environment script in TRL includes inline dependency metadata (PEP 723) so you can also run them directly with <a href="https://docs.astral.sh/uv/" rel="nofollow">uv</a>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->uv run examples/scripts/openenv/echo.py<!----></pre></div><!----> <p>This automatically installs the required environment package in an isolated virtual environment.</p></blockquote> <!--[1--><h2 class="relative group"><a id="quick-start" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quick-start"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Quick start</span></h2><!--]--><!----> <p>The fastest way to understand the integration is a complete example. The <a href="https://github.com/huggingface/trl/blob/main/examples/scripts/openenv/echo.py" rel="nofollow">echo.py</a> script trains a model with the <a href="https://huggingface.co/docs/openenv/environments/echo" rel="nofollow">Echo environment</a>, which rewards completions based on their text length:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset | |
| <span class="hljs-keyword">from</span> echo_env <span class="hljs-keyword">import</span> EchoEnv | |
| <span class="hljs-keyword">from</span> echo_env.models <span class="hljs-keyword">import</span> EchoAction | |
| <span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig, GRPOTrainer | |
| ENV_URL = <span class="hljs-string">"https://openenv-echo-env.hf.space"</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">EchoToolEnv</span>: | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>): | |
| <span class="hljs-variable language_">self</span>.env = EchoEnv(base_url=ENV_URL) | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, **kwargs</span>) -> <span class="hljs-built_in">str</span> | <span class="hljs-literal">None</span>: | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span> | |
| <span class="hljs-keyword">return</span> <span class="hljs-literal">None</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">echo</span>(<span class="hljs-params">self, message: <span class="hljs-built_in">str</span></span>) -> <span class="hljs-built_in">str</span>: | |
| <span class="hljs-string">""" | |
| Echo the message back from the environment. | |
| Args: | |
| message: The message to echo | |
| Returns: | |
| The echoed message. | |
| """</span> | |
| observation = <span class="hljs-variable language_">self</span>.env.step(EchoAction(message=message)) | |
| <span class="hljs-variable language_">self</span>.reward = observation.observation.reward | |
| <span class="hljs-keyword">return</span> observation.observation.echoed_message | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">environments, **kwargs</span>): | |
| <span class="hljs-keyword">return</span> [env.reward <span class="hljs-keyword">for</span> env <span class="hljs-keyword">in</span> environments] | |
| dataset = Dataset.from_dict( | |
| {<span class="hljs-string">"prompt"</span>: [[{<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Try to echo 'Hello World!' in the environment."</span>}]] * <span class="hljs-number">64</span>} | |
| ) | |
| trainer = GRPOTrainer( | |
| model=<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>, | |
| train_dataset=dataset, | |
| reward_funcs=reward_func, | |
| args=GRPOConfig( | |
| chat_template_kwargs={<span class="hljs-string">"enable_thinking"</span>: <span class="hljs-literal">False</span>}, | |
| log_completions=<span class="hljs-literal">True</span>, | |
| ), | |
| environment_factory=EchoToolEnv, | |
| ) | |
| trainer.train()<!----></pre></div><!----> <p>That’s it. Here’s what happens under the hood:</p> <ol><li><strong><code>environment_factory=EchoToolEnv</code></strong>: The trainer creates one <code>EchoToolEnv</code> instance per generation (pass the class, not an instance).</li> <li><strong><code>reset()</code></strong> is called at the start of each episode to initialize state. Returns an observation string (or <code>None</code>).</li> <li><strong>Tool discovery</strong>: The trainer discovers all public methods on the environment instance (here, <code>echo()</code>) and exposes them as function-calling tools. Each method must have a proper docstring with typed arguments, which the trainer uses to build the tool schema.</li> <li><strong>Multi-turn loop</strong>: The trainer generates a completion, parses tool calls, executes <code>echo()</code>, appends the result, and generates again, until the model stops calling tools or <code>max_completion_length</code> is reached.</li> <li><strong>Reward function</strong>: Reads <code>env.reward</code> from each environment instance after the episode (before the environment is reset).</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># Run the example</span> | |
| python examples/scripts/openenv/echo.py | |
| <span class="hljs-comment"># Customize model and environment URL</span> | |
| python examples/scripts/openenv/echo.py --model Qwen/Qwen3-0.6B --env-host https://openenv-echo-env.hf.space<!----></pre></div><!----> <p>Below is the reward curve from training:</p> <iframe src="https://trl-lib-trackio.hf.space?project=openenv&metrics=train/rewards/reward_from_env/mean&runs=qgallouedec-1761202871&sidebar=hidden&navbar=hidden" style="width:100%; max-width:800px; height:500px; border:0;"></iframe> <blockquote class="note"><p>You can explore more ready-to-use example <a href="example_overview#openenv-scripts">scripts</a> and <a href="example_overview#openenv-notebooks">notebooks</a> in the Examples Overview.</p></blockquote> <!--[1--><h2 class="relative group"><a id="how-environmentfactory-works" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-environmentfactory-works"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>How environment_factory works</span></h2><!--]--><!----> <p>TRL’s <a href="/docs/trl/main/en/gspo_token#trl.GRPOTrainer">GRPOTrainer</a> supports interactive environment training through the <code>environment_factory</code> argument. When provided, the trainer automatically handles the multi-turn tool-calling loop: it generates completions, parses tool calls, executes them against the environment, and feeds the results back to the model. All without custom rollout code.</p> <!--[2--><h3 class="relative group"><a id="environment-class-requirements" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#environment-class-requirements"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Environment class requirements</span></h3><!--]--><!----> <p>Your environment class must follow these rules:</p> <ul><li><code>__init__(self)</code> <em>(optional)</em>: If provided, must take no arguments. Use it to initialize state or clients. If you need external configuration (e.g., a URL), capture it from the enclosing scope or module-level variables.</li> <li><code>reset(self, **kwargs)</code>: Called at the start of each episode. Receives all dataset columns as keyword arguments. Return a string observation (or <code>None</code> for no initial observation).</li> <li><strong>Tool methods</strong>: Any public method (not starting with <code>_</code>) other than <code>reset</code> is automatically exposed as a tool. Each tool method must have a docstring with <code>Args:</code> descriptions, since the trainer uses these to generate the tool schema for the model.</li></ul> <!--[2--><h3 class="relative group"><a id="tips-for-environment-classes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tips-for-environment-classes"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Tips for environment classes</span></h3><!--]--><!----> <ul><li><strong>State for reward</strong>: You can store any state you want on the environment instance (e.g., <code>self.reward</code>, <code>self.done</code>, etc.) and access it in your reward function via the <code>environments</code> parameter. Refer to the <a href="#quick-start">Quick Start guide</a> for an example of this pattern.</li> <li><strong>Error handling</strong>: If a tool method raises an exception (e.g., <code>ValueError("Game over.")</code>), the trainer catches it and feeds the error message back to the model as a tool response. This is the recommended way to signal that an action is invalid or that the episode has ended.</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->ENV_URL = <span class="hljs-string">"https://my-env.hf.space"</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyEnv</span>: | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>): | |
| <span class="hljs-variable language_">self</span>.client = MyClient(base_url=ENV_URL) <span class="hljs-comment"># captured from enclosing scope</span> | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, **kwargs</span>) -> <span class="hljs-built_in">str</span> | <span class="hljs-literal">None</span>: | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span> | |
| <span class="hljs-keyword">return</span> <span class="hljs-string">"Initial observation for the model"</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">my_tool</span>(<span class="hljs-params">self, arg1: <span class="hljs-built_in">str</span>, arg2: <span class="hljs-built_in">int</span></span>) -> <span class="hljs-built_in">str</span>: | |
| <span class="hljs-string">""" | |
| Description of what this tool does. | |
| Args: | |
| arg1: Description of arg1 | |
| arg2: Description of arg2 | |
| Returns: | |
| The result message. | |
| """</span> | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">1.0</span> | |
| <span class="hljs-keyword">return</span> <span class="hljs-string">"Tool result"</span><!----></pre></div><!----> <blockquote class="important"><p>Tools must be <strong>individual methods</strong> with descriptive names and typed arguments (e.g., <code>guess(word: str)</code>, <code>move(direction: str)</code>). We do not recommend using generic methods like <code>step(action)</code>, since the model needs meaningful tool names and argument descriptions to learn tool calling.</p></blockquote> <!--[2--><h3 class="relative group"><a id="reward-functions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#reward-functions"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Reward functions</span></h3><!--]--><!----> <p>Reward functions receive the <code>environments</code> parameter (a list of environment instances), so you can access any state stored during the episode:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">environments, **kwargs</span>) -> <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]: | |
| <span class="hljs-keyword">return</span> [env.reward <span class="hljs-keyword">for</span> env <span class="hljs-keyword">in</span> environments]<!----></pre></div><!----> <p>For more information on reward functions, see the <a href="grpo_trainer#using-a-custom-reward-function">GRPO - Custom Reward Functions</a>.</p> <!--[2--><h3 class="relative group"><a id="tips-for-reward-functions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tips-for-reward-functions"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Tips for reward functions</span></h3><!--]--><!----> <p>A few things we’ve found helpful when working with OpenEnv environments and GRPO:</p> <ul><li><strong>Simple rewards work well.</strong> In our experiments with Wordle and Sudoku, binary rewards (1.0 for success, 0.0 otherwise) gave cleaner training signals than shaped rewards with partial credit. GRPO compares completions within a group, so the relative ranking matters more than the absolute values.</li> <li><strong>Check the final state, not the path.</strong> When possible, let the environment judge the outcome (e.g., “did the model solve the puzzle?”) rather than checking if it followed a specific sequence of actions. This gives the model freedom to discover its own strategies.</li> <li><strong>Test your reward before training.</strong> Run a few episodes manually (see the <a href="https://github.com/huggingface/trl/blob/main/examples/notebooks/openenv_wordle_grpo.ipynb" rel="nofollow">Wordle example notebook</a>) to confirm the environment returns sensible rewards. If a capable model can’t score higher than a random baseline, the reward signal may need adjustment.</li></ul> <!--[2--><h3 class="relative group"><a id="maxcompletionlength-in-multi-turn-episodes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#maxcompletionlength-in-multi-turn-episodes"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>max_completion_length in multi-turn episodes</span></h3><!--]--><!----> <p>The <code>max_completion_length</code> parameter limits the <strong>total number of tokens across the entire multi-turn conversation</strong> (all model generations + tool results combined), not just a single generation. For environments with many turns (e.g., Sudoku with dozens of moves), you may need to increase it:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->args = GRPOConfig( | |
| max_completion_length=<span class="hljs-number">4096</span>, <span class="hljs-comment"># default is usually 256-1024, increase for long episodes</span> | |
| <span class="hljs-comment"># ...</span> | |
| )<!----></pre></div><!----> <p>If episodes are being cut short (model stops mid-game), this is likely the cause.</p> <!--[1--><h2 class="relative group"><a id="advanced-example-wordle" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#advanced-example-wordle"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Advanced example: Wordle</span></h2><!--]--><!----> <p>Let’s train a model to play <a href="https://www.nytimes.com/games/wordle/index.html" rel="nofollow">Wordle</a> using the <a href="https://huggingface.co/docs/openenv/environments/textarena" rel="nofollow"><code>TextArena</code></a> environment. This demonstrates multi-turn interaction, cumulative feedback handling, and episode termination via exceptions.</p> <blockquote class="note"><p>You can explore the notebook version of this example in <a href="https://github.com/huggingface/trl/blob/main/examples/notebooks/openenv_wordle_grpo.ipynb" rel="nofollow">the OpenEnv Wordle GRPO example</a>.</p></blockquote> <!--[2--><h3 class="relative group"><a id="the-textarena-environment" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#the-textarena-environment"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>The TextArena Environment</span></h3><!--]--><!----> <p><a href="https://huggingface.co/papers/2504.11442" rel="nofollow">TextArena</a> is an open-source collection of competitive text-based games designed to evaluate reasoning skills in LLMs using textual games like Wordle, Snake, Tic-Tac-Toe, and more.</p> <p><img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/text_arena_evals.png" alt="image of TextArena"/></p> <!--[2--><h3 class="relative group"><a id="why-wordle" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#why-wordle"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Why Wordle?</span></h3><!--]--><!----> <p>Wordle is a good benchmark for environment-based RL because it requires reasoning about feedback, is purely text-based, and models from 1B parameters can improve at it. Each guess is only 8 tokens, making it lightweight to experiment with.</p> <blockquote class="note"><p>How does Wordle work? | |
| Wordle is a word guessing game where the player has to guess a 5-letter word in 6 attempts. After each guess, the environment provides letter-by-letter feedback:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class=" "><!----><span class="hljs-selector-tag">G</span> U E S S | |
| <span class="hljs-attribute">X</span> <span class="hljs-selector-tag">G</span> <span class="hljs-attribute">Y</span> <span class="hljs-attribute">X</span> <span class="hljs-attribute">X</span><!----></pre></div><!----> <p>X = not in the word, G = correct position (green), Y = wrong position (yellow). Here, “U” is correct and in place, “E” is in the word but misplaced.</p></blockquote> <!--[2--><h3 class="relative group"><a id="environment-class" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#environment-class"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Environment class</span></h3><!--]--><!----> <p>The <code>WordleEnv</code> class wraps the TextArena client and exposes <code>guess()</code> as the tool:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> textarena_env <span class="hljs-keyword">import</span> TextArenaAction, TextArenaEnv | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">WordleEnv</span>: | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>): | |
| <span class="hljs-variable language_">self</span>.client = TextArenaEnv(base_url=<span class="hljs-string">"https://openenv-wordle.hf.space"</span>) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, **kwargs</span>) -> <span class="hljs-built_in">str</span> | <span class="hljs-literal">None</span>: | |
| result = <span class="hljs-variable language_">self</span>.client.reset() | |
| <span class="hljs-variable language_">self</span>._last_full_feedback = result.observation.messages[<span class="hljs-number">0</span>].content | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span> | |
| <span class="hljs-variable language_">self</span>.done = <span class="hljs-literal">False</span> | |
| <span class="hljs-keyword">return</span> <span class="hljs-variable language_">self</span>._last_full_feedback | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">guess</span>(<span class="hljs-params">self, guess: <span class="hljs-built_in">str</span></span>) -> <span class="hljs-built_in">str</span>: | |
| <span class="hljs-string">""" | |
| Make a guess in the Wordle environment. | |
| Args: | |
| guess: The guessed word, formatted as '[abcde]' | |
| Returns: | |
| The feedback message from the environment. | |
| """</span> | |
| <span class="hljs-keyword">if</span> <span class="hljs-variable language_">self</span>.done: | |
| <span class="hljs-keyword">raise</span> ValueError(<span class="hljs-string">"Game over."</span>) | |
| result = <span class="hljs-variable language_">self</span>.client.step(TextArenaAction(message=guess)) | |
| _full_feedback = result.observation.messages[<span class="hljs-number">0</span>].content | |
| feedback = _full_feedback[<span class="hljs-built_in">len</span>(<span class="hljs-variable language_">self</span>._last_full_feedback):] | |
| <span class="hljs-variable language_">self</span>._last_full_feedback = _full_feedback | |
| <span class="hljs-keyword">if</span> <span class="hljs-string">"You attempted an invalid move"</span> <span class="hljs-keyword">in</span> feedback: | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span> | |
| <span class="hljs-keyword">else</span>: | |
| <span class="hljs-variable language_">self</span>.reward = result.reward | |
| <span class="hljs-variable language_">self</span>.done = result.done | |
| <span class="hljs-keyword">return</span> feedback<!----></pre></div><!----> <p>Key design choices:</p> <ul><li><strong><code>reset()</code></strong> returns the initial game message as the first observation the model sees.</li> <li><strong><code>guess()</code></strong> is the only tool. The model calls it each turn with a 5-letter word.</li> <li><strong>Cumulative feedback slicing</strong>: TextArena returns the full game history each turn. We slice out only the new part to avoid repeating context.</li> <li><strong>Exception on done</strong>: If the model tries to guess after the game ends, <code>guess()</code> raises a <code>ValueError</code>. The trainer catches this and feeds <code>"Game over."</code> back to the model as a tool response. The model learns to stop calling tools after this signal.</li></ul> <!--[2--><h3 class="relative group"><a id="reward-function-and-training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#reward-function-and-training"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Reward function and training</span></h3><!--]--><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset | |
| <span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig, GRPOTrainer | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">environments, **kwargs</span>) -> <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]: | |
| <span class="hljs-keyword">return</span> [env.reward <span class="hljs-keyword">for</span> env <span class="hljs-keyword">in</span> environments] | |
| prompt = <span class="hljs-string">"""You are an expert Wordle solver with deep knowledge of English vocabulary... | |
| Use the tool `guess` to make a guess."""</span> | |
| dataset = Dataset.from_dict({<span class="hljs-string">"prompt"</span>: [[{<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: prompt}]] * <span class="hljs-number">1000</span>}) | |
| trainer = GRPOTrainer( | |
| model=<span class="hljs-string">"Qwen/Qwen3-1.7B"</span>, | |
| reward_funcs=reward_func, | |
| train_dataset=dataset, | |
| args=GRPOConfig( | |
| use_vllm=<span class="hljs-literal">True</span>, | |
| vllm_mode=<span class="hljs-string">"colocate"</span>, | |
| chat_template_kwargs={<span class="hljs-string">"enable_thinking"</span>: <span class="hljs-literal">False</span>}, | |
| max_completion_length=<span class="hljs-number">1024</span>, | |
| num_generations=<span class="hljs-number">4</span>, | |
| gradient_accumulation_steps=<span class="hljs-number">64</span>, | |
| ), | |
| environment_factory=WordleEnv, | |
| ) | |
| trainer.train()<!----></pre></div><!----> <p>The environment returns <code>1.0</code> if the model wins and <code>0.0</code> otherwise.</p> <!--[2--><h3 class="relative group"><a id="running-the-example" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#running-the-example"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Running the example</span></h3><!--]--><!----> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">colocate</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">server</div><!--]--></div> <div class="language-select"><!--[0--><p><strong>Colocate mode (1 GPU, recommended)</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->python examples/scripts/openenv/wordle.py --vllm-mode colocate<!----></pre></div><!----> <p>This runs vLLM in the same process as training, requiring only a single GPU.</p><!----><!--]--><!----> <!--[-1--><!--]--><!----><!----></div><!----> <!--[2--><h3 class="relative group"><a id="results" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#results"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Results</span></h3><!--]--><!----> <p>The model improves its performance by reducing repetitions and increasing correct guesses. However, Qwen3-1.7B with <code>enable_thinking=False</code> is not able to consistently win the game.</p> <iframe src="https://burtenshaw-wordle-grpo.hf.space?project=group-Qwen-Qwen3-17B&metrics=reward&runs=run-2025-10-26_09-39-49,run-2025-10-26_08-04-49&sidebar=hidden&navbar=hidden" style="width:100%; max-width:800px; height:500px; border:0;"></iframe> <blockquote class="note"><p>With <code>enable_thinking=False</code> (the default in these examples), small models like Qwen3-1.7B can learn to improve their guesses but should not be expected to consistently solve the game. For significantly better results, use larger models or enable thinking mode (<code>enable_thinking=True</code>), which allows the model to reason before making a guess at the cost of longer completions.</p></blockquote> <p>We experimented with larger models like <a href="https://huggingface.co/openai/gpt-oss-20b" rel="nofollow"><code>gpt-oss-20b</code></a> and found that it was able to consistently win the game, though this requires significantly more compute.</p> <!--[1--><h2 class="relative group"><a id="multi-environment-training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#multi-environment-training"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Multi-environment training</span></h2><!--]--><!----> <p>You can train a single model across multiple environments simultaneously. This is useful when you want a model to learn different skills in parallel. For example, playing Wordle (language reasoning) and Catch (spatial reasoning) in the same training run.</p> <p>The key idea is to create a <strong>meta-environment class</strong> that wraps multiple environments and routes each sample to the correct one using a dataset column.</p> <!--[2--><h3 class="relative group"><a id="how-it-works" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-it-works"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>How it works</span></h3><!--]--><!----> <ol><li>Add an <code>"env"</code> column (or similar) to your dataset that identifies which environment each sample belongs to.</li> <li>In <code>reset(**kwargs)</code>, read <code>kwargs["env"]</code> to select the active environment for that episode.</li> <li>Expose tools from all environments; the trainer discovers all public methods.</li> <li>Use separate reward functions per environment, returning <code>None</code> for samples that don’t belong to that environment. TRL handles <code>None</code> values with <code>nansum</code>/<code>nanmean</code>.</li></ol> <!--[2--><h3 class="relative group"><a id="example-wordle--catch" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#example-wordle--catch"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Example: Wordle + Catch</span></h3><!--]--><!----> <p>The <a href="https://github.com/huggingface/trl/blob/main/examples/scripts/openenv/multi_env.py" rel="nofollow">multi_env.py</a> script trains on Wordle and Catch simultaneously:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">class</span> <span class="hljs-title class_">MultiEnv</span>: | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>): | |
| <span class="hljs-variable language_">self</span>._wordle_client = <span class="hljs-literal">None</span> | |
| <span class="hljs-variable language_">self</span>._catch_client = <span class="hljs-literal">None</span> | |
| <span class="hljs-variable language_">self</span>.active = <span class="hljs-literal">None</span> | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span> | |
| <span class="hljs-variable language_">self</span>.done = <span class="hljs-literal">False</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, **kwargs</span>) -> <span class="hljs-built_in">str</span> | <span class="hljs-literal">None</span>: | |
| <span class="hljs-variable language_">self</span>.active = kwargs.get(<span class="hljs-string">"env"</span>, <span class="hljs-string">"wordle"</span>) | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span> | |
| <span class="hljs-variable language_">self</span>.done = <span class="hljs-literal">False</span> | |
| <span class="hljs-keyword">if</span> <span class="hljs-variable language_">self</span>.active == <span class="hljs-string">"wordle"</span>: | |
| <span class="hljs-keyword">if</span> <span class="hljs-variable language_">self</span>._wordle_client <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">try</span>: | |
| <span class="hljs-variable language_">self</span>._wordle_client.close() | |
| <span class="hljs-keyword">except</span> Exception: | |
| <span class="hljs-keyword">pass</span> | |
| <span class="hljs-variable language_">self</span>._wordle_client = TextArenaEnv(base_url=WORDLE_URL) | |
| result = <span class="hljs-variable language_">self</span>._wordle_client.reset() | |
| <span class="hljs-variable language_">self</span>._last_full_feedback = result.observation.messages[<span class="hljs-number">0</span>].content | |
| <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span> | |
| <span class="hljs-keyword">return</span> <span class="hljs-variable language_">self</span>._last_full_feedback | |
| <span class="hljs-keyword">elif</span> <span class="hljs-variable language_">self</span>.active == <span class="hljs-string">"catch"</span>: | |
| <span class="hljs-keyword">if</span> <span class="hljs-variable language_">self</span>._catch_client <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">try</span>: | |
| <span class="hljs-variable language_">self</span>._catch_client.close() | |
| <span class="hljs-keyword">except</span> Exception: | |
| <span class="hljs-keyword">pass</span> | |
| <span class="hljs-variable language_">self</span>._catch_client = OpenSpielEnv(base_url=CATCH_URL) | |
| result = <span class="hljs-variable language_">self</span>._catch_client.reset() | |
| <span class="hljs-variable language_">self</span>.done = result.observation.done | |
| <span class="hljs-keyword">return</span> _format_catch_obs(result.observation.info_state) | |
| <span class="hljs-comment"># Wordle tool</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">guess</span>(<span class="hljs-params">self, guess: <span class="hljs-built_in">str</span></span>) -> <span class="hljs-built_in">str</span>: | |
| <span class="hljs-string">"""Make a guess in the Wordle environment. ..."""</span> | |
| ... | |
| <span class="hljs-comment"># Catch tools</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">move</span>(<span class="hljs-params">self, direction: <span class="hljs-built_in">str</span></span>) -> <span class="hljs-built_in">str</span>: | |
| <span class="hljs-string">"""Move the paddle left or right. ..."""</span> | |
| ... | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">stay</span>(<span class="hljs-params">self</span>) -> <span class="hljs-built_in">str</span>: | |
| <span class="hljs-string">"""Do nothing and let the ball fall one step. ..."""</span> | |
| ...<!----></pre></div><!----> <p>Key patterns:</p> <ul><li><strong>Lazy client initialization</strong>: Create clients in <code>reset()</code>, not <code>__init__()</code>, to avoid unnecessary WebSocket connections.</li> <li><strong>Close before reopen</strong>: Close the previous client before creating a new one to avoid server capacity errors.</li> <li><strong><code>kwargs</code> routing</strong>: The <code>"env"</code> column from the dataset is passed to <code>reset()</code> as a keyword argument.</li> <li><strong>All tools are exposed simultaneously</strong>: The model sees <code>guess</code>, <code>move</code>, and <code>stay</code> as available tools regardless of the active environment. If it calls the wrong tool (e.g., <code>move</code> during Wordle), the method raises a <code>ValueError</code> that the trainer catches gracefully. In practice, models learn to use the correct tools based on the system prompt.</li></ul> <!--[2--><h3 class="relative group"><a id="per-environment-reward-functions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#per-environment-reward-functions"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Per-environment reward functions</span></h3><!--]--><!----> <p>Each reward function returns <code>None</code> for samples from other environments:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">def</span> <span class="hljs-title function_">wordle_reward</span>(<span class="hljs-params">environments, **kwargs</span>) -> <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span> | <span class="hljs-literal">None</span>]: | |
| <span class="hljs-keyword">return</span> [env.reward <span class="hljs-keyword">if</span> env.active == <span class="hljs-string">"wordle"</span> <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">for</span> env <span class="hljs-keyword">in</span> environments] | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">catch_reward</span>(<span class="hljs-params">environments, **kwargs</span>) -> <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span> | <span class="hljs-literal">None</span>]: | |
| rewards = [] | |
| <span class="hljs-keyword">for</span> env <span class="hljs-keyword">in</span> environments: | |
| <span class="hljs-keyword">if</span> env.active != <span class="hljs-string">"catch"</span>: | |
| rewards.append(<span class="hljs-literal">None</span>) | |
| <span class="hljs-keyword">elif</span> env.done: | |
| rewards.append(<span class="hljs-built_in">max</span>(env.reward, <span class="hljs-number">0.0</span>)) | |
| <span class="hljs-keyword">else</span>: | |
| rewards.append(<span class="hljs-number">0.0</span>) | |
| <span class="hljs-keyword">return</span> rewards<!----></pre></div><!----> <p>TRL converts <code>None</code> to <code>nan</code> internally and uses <code>nansum</code>/<code>nanmean</code> for aggregation, so each sample is only scored by its relevant reward function.</p> <!--[2--><h3 class="relative group"><a id="dataset-with-environment-routing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dataset-with-environment-routing"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Dataset with environment routing</span></h3><!--]--><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->n = <span class="hljs-number">500</span> | |
| dataset = Dataset.from_dict({ | |
| <span class="hljs-string">"prompt"</span>: ( | |
| [[{<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: wordle_prompt}]] * n | |
| + [[{<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: catch_prompt}]] * n | |
| ), | |
| <span class="hljs-string">"env"</span>: [<span class="hljs-string">"wordle"</span>] * n + [<span class="hljs-string">"catch"</span>] * n, | |
| })<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="running-the-multi-environment-example" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#running-the-multi-environment-example"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Running the multi-environment example</span></h3><!--]--><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->python examples/scripts/openenv/multi_env.py \ | |
| --wordle-url https://openenv-wordle.hf.space \ | |
| --catch-url https://openenv-openspiel-env.hf.space \ | |
| --vllm-mode colocate \ | |
| --gradient-accumulation-steps 4 \ | |
| --num-generations 8<!----></pre></div><!----> <blockquote class="tip"><p>When training across multiple environments, monitor the per-reward-function metrics (<code>train/reward_func_0</code>, <code>train/reward_func_1</code>, etc.) rather than the combined <code>train/reward</code>. The combined metric alternates between environments and can appear noisy.</p></blockquote> <!--[1--><h2 class="relative group"><a id="running-the-environments" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#running-the-environments"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Running the environments</span></h2><!--]--><!----> <p>When using <code>environment_factory</code>, the trainer connects to the environment server automatically. You just need the server to be running. There are three ways to run an OpenEnv environment server:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">space</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">docker</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">local</div><!--]--></div> <div class="language-select"><!--[0--><p><strong>Connect to a remote Hugging Face Space</strong> <em>(simplest)</em></p> <p>Most example scripts default to a hosted Space (no setup needed):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->env = EchoEnv(base_url=<span class="hljs-string">"https://openenv-echo-env.hf.space"</span>)<!----></pre></div><!----> <blockquote class="warning"><p>For training, <strong>duplicate the Space to your own account</strong> to avoid concurrency issues. The trainer opens N simultaneous WebSocket connections (one per generation), and shared Spaces may not support this. See <a href="#server-concurrency">Server concurrency</a> for details.</p></blockquote><!----><!--]--><!----> <!--[-1--><!--]--><!----> <!--[-1--><!--]--><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="environments-catalog" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#environments-catalog"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Environments catalog</span></h2><!--]--><!----> <p>The best way to explore the current catalog of maintained environments is by visiting the official OpenEnv <a href="https://huggingface.co/collections/openenv/environment-hub" rel="nofollow">catalog</a>.</p> <p>To create your own environment, check out the guide on <a href="https://huggingface.co/docs/openenv/getting_started/environment-builder" rel="nofollow">Building Your Own Environment with OpenEnv</a>. Environments are tightly integrated with the Hub, so you can push new environments for the community to reuse.</p> <!--[1--><h2 class="relative group"><a id="server-concurrency" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#server-concurrency"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Server concurrency</span></h2><!--]--><!----> <p>When using <code>environment_factory</code>, the trainer creates N environment instances (one per generation), each opening a WebSocket connection to the server. By default, OpenEnv servers allow only 1 concurrent session, which will cause failures during training.</p> <p>To support parallel training, configure the server for concurrency:</p> <ol><li>In your environment file, declare concurrent session support:</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->SUPPORTS_CONCURRENT_SESSIONS: <span class="hljs-built_in">bool</span> = <span class="hljs-literal">True</span><!----></pre></div><!----> <ol start="2"><li>In your server app, set the concurrency limit:</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->app = create_app( | |
| create_my_environment, | |
| MyAction, | |
| MyObservation, | |
| max_concurrent_envs=<span class="hljs-number">64</span>, <span class="hljs-comment"># match or exceed generation_batch_size</span> | |
| )<!----></pre></div><!----> <blockquote class="tip"><p><code>max_concurrent_envs</code> should be ≥ <code>generation_batch_size</code> (which defaults to <code>per_device_train_batch_size × gradient_accumulation_steps</code>). For example, with <code>gradient_accumulation_steps=64</code> and batch size 1, you need at least 64 concurrent sessions.</p></blockquote> <!--[1--><h2 class="relative group"><a id="environmentfactory-vs-rolloutfunc" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#environmentfactory-vs-rolloutfunc"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>environment_factory vs rollout_func</span></h2><!--]--><!----> <p><a href="/docs/trl/main/en/gspo_token#trl.GRPOTrainer">GRPOTrainer</a> supports two approaches for environment-based training:</p> <ul><li><strong><code>environment_factory</code></strong> (recommended): You define an environment class with tool methods, and the trainer handles generation, tool-call parsing, and the multi-turn loop automatically. This is the approach used throughout this guide.</li> <li><strong><code>rollout_func</code></strong>: You write the entire generation and environment interaction loop yourself. This gives full control over how completions are produced, how tools are executed, and how rewards are computed.</li></ul> <p>Use <code>rollout_func</code> when <code>environment_factory</code> doesn’t fit your use case. For example, <strong>external agent servers</strong> where an external server owns the generation loop and manages its own agent-environment interaction protocol.</p> <!--[1--><h2 class="relative group"><a id="training-on-harnesses-training-a-real-coding-agent-opencode" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#training-on-harnesses-training-a-real-coding-agent-opencode"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Training on harnesses: training a real coding agent (opencode)</span></h2><!--]--><!----> <p>The integrations above are <strong>white-box</strong>: TRL drives the multi-turn loop itself. It samples each turn, parses the tool calls, runs them, and feeds the results back.</p> <p>Some agents cannot be driven this way because they own their own loop. A production coding agent harness like <a href="https://opencode.ai" rel="nofollow"><code>opencode</code></a> has its own planner, tool set, context management, and stop condition. You want to train that exact agent, not a reimplementation of it.</p> <p>For this, TRL provides an experimental <strong>black box (loop-owning)</strong> path built on <a href="/docs/trl/main/en/async_grpo_trainer#trl.experimental.async_grpo.AsyncGRPOTrainer">experimental.async_grpo.AsyncGRPOTrainer</a> and a <code>HarnessRolloutWorker</code> specific for OpenEnv that drives an <a href="https://huggingface.co/docs/openenv" rel="nofollow">OpenEnv <code>ResourceSessionFactory</code></a>. See <a href="https://github.com/huggingface/trl/blob/main/examples/scripts/openenv/opencode.py" rel="nofollow"><code>examples/scripts/openenv/opencode.py</code></a> for a complete, self-contained example. To scale rollouts beyond a single node, <a href="https://github.com/huggingface/trl/blob/main/examples/scripts/openenv/opencode_hf_sandbox.py" rel="nofollow"><code>examples/scripts/openenv/opencode_hf_sandbox.py</code></a> runs each rollout in its own remote Hugging Face sandbox instead of a local subprocess.</p> <!--[2--><h3 class="relative group"><a id="how-it-works" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-it-works"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>How it works</span></h3><!--]--><!----> <p>TRL does not sample each turn here. The agent runs to completion on its own, and TRL reads back what it did:</p> <ol><li>The agent runs inside an OpenEnv session (a local subprocess sandbox in the example, so no container is needed) in <code>transparent_proxy</code> mode. A small proxy inside the sandbox forwards the agent’s <code>/v1/chat/completions</code> calls to your vLLM server and records each turn’s token ids and logprobs.</li> <li>When the agent stops, TRL reads the proxy trace, rebuilds the per-turn training rows from the recorded ids, and scores the final workspace with the session’s <code>verify()</code> method (a held-out verifier).</li> <li>GRPO trains on those rows. The reward is propagated to every trained token through the group-relative advantage.</li></ol> <p>Each rollout runs in its own isolated session. In the example that means one sandbox directory, one proxy on its own port, and one agent process per rollout. The isolation matters for two reasons: the proxy has to capture exactly that rollout’s tokens, and one rollout must not interfere with another. The <code>max_inflight_tasks</code> setting controls how many rollouts run at the same time.</p> <!--[2--><h3 class="relative group"><a id="wiring" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#wiring"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Wiring</span></h3><!--]--><!----> <p>You pass a <code>HarnessRolloutWorker</code> to <a href="/docs/trl/main/en/async_grpo_trainer#trl.experimental.async_grpo.AsyncGRPOTrainer">experimental.async_grpo.AsyncGRPOTrainer</a> with <code>harness_adapter=None</code> to select loop-owning mode. Besides the usual training arguments, you provide three functions (<code>rollout_reward_fn</code>, <code>train_turn_fn</code>, and <code>agent_turn_fn</code>) that tell TRL how to score, filter, and read the agent’s rollouts. They are described in <a href="#what-you-need-to-define">What you need to define</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> trl.experimental.async_grpo <span class="hljs-keyword">import</span> AsyncGRPOConfig, AsyncGRPOTrainer | |
| <span class="hljs-keyword">from</span> trl.experimental.async_grpo.openenv_harness <span class="hljs-keyword">import</span> HarnessRolloutWorker, has_tool_call | |
| worker = HarnessRolloutWorker( | |
| harness_session_factory=build_factory(...), <span class="hljs-comment"># your OpenEnv ResourceSessionFactory</span> | |
| harness_adapter=<span class="hljs-literal">None</span>, <span class="hljs-comment"># loop-owning: the agent runs its own loop</span> | |
| rollout_reward_fn=my_reward, <span class="hljs-comment"># outcome -> float | None</span> | |
| train_turn_fn=has_tool_call, <span class="hljs-comment"># reinforce only action turns</span> | |
| agent_turn_fn=my_agent_turns, <span class="hljs-comment"># drop auxiliary (non-agent) calls</span> | |
| model_name=model, | |
| dataset=dataset, | |
| reward_funcs=[], | |
| processing_class=tokenizer, | |
| num_generations=<span class="hljs-number">8</span>, | |
| max_inflight_tasks=<span class="hljs-number">8</span>, | |
| vllm_server_url=vllm_url, | |
| ) | |
| trainer = AsyncGRPOTrainer(model=model, args=config, train_dataset=dataset, rollout_worker=worker) | |
| trainer.train()<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="what-you-need-to-define" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#what-you-need-to-define"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>What you need to define</span></h3><!--]--><!----> <p>In loop-owning mode the agent runs the loop and TRL only sees a raw trace of the LLM calls it made. TRL takes care of everything mechanical: it drives the agent, captures the trace, rebuilds the training rows from the recorded token ids, keeps the policy in sync with vLLM, and runs the GRPO update. But three decisions depend on your task and your agent, and TRL cannot make them for you. You supply each as a small function. Every function receives a typed input and has a default, so you only write the ones your task needs.</p> <!--[3--><h4 class="relative group"><a id="rolloutrewardfn--how-to-score-a-rollout" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rolloutrewardfn--how-to-score-a-rollout"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>rollout_reward_fn : how to score a rollout</span></h4><!--]--><!----> <p><code>Callable[[HarnessRolloutOutcome], float | None]</code></p> <p>TRL does not know what success means for your task, so you turn each finished rollout into a scalar reward. The function receives a <code>HarnessRolloutOutcome</code> that describes what the agent did:</p> <ul><li><code>env_reward</code> (<code>float | None</code>): the reward from the session’s <code>verify()</code> (for opencode, the fraction of held-out tests that passed), or <code>None</code> when the rollout could not be scored.</li> <li><code>completion</code> (<code>list[dict]</code>): the final message transcript.</li> <li><code>trace</code> (<code>list[TraceEntry]</code>): the raw proxy trace.</li> <li><code>tool_call_count</code> and <code>tool_failure_count</code> (<code>int</code>): how many tool calls the agent made, and how many looked like failures.</li> <li><code>tool_calls_by_name</code> (<code>dict[str, int]</code>): calls per tool, for example <code>{"bash": 3, "edit": 2}</code>.</li> <li><code>timed_out</code> (<code>bool</code>): whether the agent ran out of its time budget.</li></ul> <p>Return a <code>float</code>, or return <code>None</code> to mark the rollout unscorable so it is dropped from the group baseline instead of being counted as a zero. If you do not pass this function, the raw <code>env_reward</code> is used as is. The opencode example turns the dense pass fraction into a binary pass or fail and subtracts small penalties for degenerate behavior, such as never running the code or looping for far too many steps.</p> <blockquote class="note"><p><strong>Why not just use <code>verify()</code>?</strong> <code>verify()</code> is the environment’s job and answers one question, “how correct was the outcome,” which keeps it clean and reusable for evaluation. The reward you train on is a separate, training-time decision (binarize the score, penalize degenerate behavior, drop unscorable rollouts). It also needs signals <code>verify()</code> never sees, since <code>verify()</code> only inspects the final workspace, while <code>rollout_reward_fn</code> also gets the trajectory (tool counts, <code>timed_out</code>, the trace). For example, a rollout can pass some tests yet never run <code>bash</code>; only <code>rollout_reward_fn</code> can see that and penalize it.</p></blockquote> <!--[3--><h4 class="relative group"><a id="trainturnfn--which-turns-to-reinforce" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#trainturnfn--which-turns-to-reinforce"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>train_turn_fn : which turns to reinforce</span></h4><!--]--><!----> <p><code>Callable[[HarnessTurn], bool]</code></p> <p>A rollout is made of many turns, and not every turn should receive a gradient. This function is called once per turn and returns <code>True</code> to train on that turn. It receives a <code>HarnessTurn</code>:</p> <ul><li><code>messages</code> (<code>list[dict]</code>): the conversation the model saw on this turn.</li> <li><code>tools</code> (<code>list[dict] | None</code>): the tools available on this turn.</li> <li><code>content</code> (<code>str</code>): the assistant’s text for this turn.</li> <li><code>tool_calls</code> (<code>list[dict]</code>): the tool calls the assistant emitted (empty for a text-only turn).</li></ul> <p>By default every agent turn is trained. For a tool-heavy agent you usually want to reinforce only the turns that took an action. TRL ships <code>has_tool_call</code> for this, which is what the opencode example uses:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">def</span> <span class="hljs-title function_">has_tool_call</span>(<span class="hljs-params">turn: HarnessTurn</span>) -> <span class="hljs-built_in">bool</span>: | |
| <span class="hljs-comment"># Train only turns where the agent actually called a tool (e.g. wrote a file, ran bash),</span> | |
| <span class="hljs-comment"># and skip pure-text turns like a final "done" message or thinking-out-loud.</span> | |
| <span class="hljs-keyword">return</span> <span class="hljs-built_in">bool</span>(turn.tool_calls)<!----></pre></div><!----> <p>An agent whose answer is plain text, such as a question-answering agent, would keep the default (train every turn) instead.</p> <!--[3--><h4 class="relative group"><a id="agentturnfn--which-trace-entries-are-real-agent-turns" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#agentturnfn--which-trace-entries-are-real-agent-turns"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>agent_turn_fn : which trace entries are real agent turns</span></h4><!--]--><!----> <p><code>Callable[[list[TraceEntry]], list[TraceEntry]]</code></p> <p>Agent frameworks make extra LLM calls that are not part of the task. opencode, for example, fires a separate call to generate a thread title and another to summarize context. These land in the same proxy trace as the real agent turns, but they must never be trained, scored, or shown as the transcript, because they answer a different prompt. This function receives the full trace and returns only the entries that are real agent turns. Each entry is a <code>TraceEntry</code>, the raw record of one LLM call (<code>request</code>, <code>response</code>, <code>completion_token_ids</code>, <code>per_token_logps</code>). By default TRL keeps every entry that has both a request and a response; the opencode example overrides this to keep only the calls that carry the agent’s own system prompt and tools, which drops the title and summary calls.</p> <!--[2--><h3 class="relative group"><a id="requirements" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#requirements"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Requirements</span></h3><!--]--><!----> <p>The vLLM server must expose tool-calling and real token ids, and enable NCCL weight sync so the agent always hits the current policy:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->vllm serve <model> \ | |
| --enable-auto-tool-choice --tool-call-parser hermes \ | |
| --logprobs-mode processed_logprobs \ | |
| --return-tokens-as-token-ids \ | |
| --weight-transfer-config <span class="hljs-string">'{"backend":"nccl"}'</span><!----></pre></div><!----> <blockquote class="note"><p>Loop-owning training lives under <code>trl.experimental</code> and its API may change. The example installs the <code>opencode</code> CLI into a sandbox template on first run (needs internet once) and uses <a href="https://huggingface.co/datasets/agentica-org/DeepCoder-Preview-Dataset" rel="nofollow"><code>agentica-org/DeepCoder-Preview-Dataset</code></a> with a held-out stdin/stdout verifier.</p></blockquote> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/trl/blob/main/docs/source/openenv.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_132ya0g = { | |
| base: "/docs/trl/main/en", | |
| assets: "/docs/trl/main/en" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/trl/main/en/_app/immutable/entry/start.8G6YWryu.js"), | |
| import("/docs/trl/main/en/_app/immutable/entry/app.B9nZyitD.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 3], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 134 kB
- Xet hash:
- 7f4904da89a923ccaad7060ed5a1ff3815e1289c6a90f6715815ea0beec17389
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.