openenv/pr_1049/en/_app/immutable/nodes/70.DbIdqBhB.js

import"../chunks/DsnmJJEf.js";import{i as k,h as E,C as Y,H as a,a as s,E as N,s as S}from"../chunks/C07d6Hje.js";import{p as F,o as z,s as l,f as H,a as _,b as x,c as R,n as q}from"../chunks/BSuxAqoA.js";const L='{"title":"OpenEnv Wordle with GRPO using TRL","local":"openenv-wordle-with-grpo-using-trl","sections":[{"title":"Install dependencies","local":"install-dependencies","sections":[{"title":"Log in to Hugging Face","local":"log-in-to-hugging-face","sections":[],"depth":3}],"depth":2},{"title":"Define the system prompt","local":"define-the-system-prompt","sections":[],"depth":2},{"title":"Define the environment","local":"define-the-environment","sections":[],"depth":2},{"title":"Define the reward function","local":"define-the-reward-function","sections":[],"depth":2},{"title":"Create the dataset","local":"create-the-dataset","sections":[],"depth":2},{"title":"Set GRPO Config","local":"set-grpo-config","sections":[],"depth":2},{"title":"Create the GRPOTrainer and start training","local":"create-the-grpotrainer-and-start-training","sections":[],"depth":2},{"title":"Save and push to Hub","local":"save-and-push-to-hub","sections":[],"depth":2},{"title":"Load the fine-tuned model and run inference","local":"load-the-fine-tuned-model-and-run-inference","sections":[],"depth":2}],"depth":1}';var K=R('<meta name="hf:doc:metadata"/>'),D=R(`<p></p> <!> <!> <p><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/notebooks/openenv_wordle_grpo.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></p> <p><img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/trl_banner_dark.png" alt="trl banner"/></p> <p>With <a href="https://github.com/huggingface/trl" rel="nofollow"><strong>Transformers Reinforcement Learning (TRL)</strong></a>, you can train a model that learns to <strong>play Wordle</strong>, a word-guessing game, through interaction and reinforcement.</p> <ul><li><a href="https://github.com/huggingface/trl" rel="nofollow">TRL GitHub Repository</a> — star us to support the project!</li> <li><a href="https://huggingface.co/docs/trl/example_overview" rel="nofollow">Official TRL Examples</a></li> <li><a href="https://huggingface.co/docs/trl/community_tutorials" rel="nofollow">Community Tutorials</a></li> <li><a href="https://github.com/huggingface/OpenEnv" rel="nofollow">OpenEnv</a></li></ul> <p>An <strong>agentic environment</strong> is a setting where a model can take actions, observe outcomes, and adjust its behavior based on feedback, similar to how humans learn from trial and error.
In this case, the agent interacts with the <strong>Wordle</strong> environment through the <a href="https://github.com/huggingface/OpenEnv" rel="nofollow"><strong>OpenEnv</strong></a> framework, which standardizes multi-agent and RL-style text environments.</p> <p><a href="https://en.wikipedia.org/wiki/Wordle" rel="nofollow">Wordle</a> is a popular word puzzle where the player must guess a secret five-letter word within six tries.
After each guess, feedback indicates whether each letter is:</p> <ul><li><strong>GREEN (G)</strong>: Correct and in the right position</li> <li><strong>YELLOW (Y)</strong>: Present but in the wrong position</li> <li><strong>GRAY (X)</strong>: Not in the word</li></ul> <p>This feedback loop makes Wordle a perfect environment for <strong>RL with LLMs</strong>, where the goal is to maximize the probability of guessing the correct word efficiently.</p> <p>We’ll fine-tune a model using <strong>GRPO</strong> (Group Relative Policy Optimization) via TRL.
Using <code>environment_factory</code>, the trainer automatically handles:</p> <ol><li>Creating environment instances for each rollout.</li> <li>Generating model completions and parsing tool calls.</li> <li>Stepping through the environment with the model’s actions.</li> <li>Collecting rewards and managing the interaction loop.</li></ol> <p>This means you only need to define the environment class and reward function — the trainer takes care of the rest.</p> <!> <p>We’ll start by installing <strong>TRL</strong> (with vLLM support), the <strong>OpenEnv</strong> Wordle environment, and <strong>trackio</strong> for logging.</p> <!> <!> <p>Log in to your <strong>Hugging Face</strong> account to save your fine-tuned model, track your experiment results directly on the Hub or access gated models. You can find your <strong>access token</strong> on your <a href="https://huggingface.co/settings/tokens" rel="nofollow">account settings page</a>.</p> <!> <!> <p>This prompt instructs the model on how to play Wordle. It includes the game rules, feedback format, and importantly, tells the model to use the <code>guess</code> tool to submit guesses. The <code>environment_factory</code> pattern uses tool calling to interact with the environment, so the model needs to know which tool to call.</p> <!> <!> <p>The <code>WordleEnv</code> class wraps the OpenEnv TextArena Wordle environment into the interface expected by <code>environment_factory</code>.</p> <p>When you pass <code>environment_factory=WordleEnv</code> to the trainer, it will:</p> <ol><li>Create a new <code>WordleEnv()</code> instance for each rollout episode.</li> <li>Call <code>reset()</code> to start a new game (returns the initial observation or <code>None</code>).</li> <li>Automatically generate model completions, parse tool calls, and invoke the corresponding methods (e.g., <code>guess(...)</code>).</li> <li>Repeat until the environment signals <code>done=True</code> or the max completion length is reached.</li></ol> <p>The environment exposes its public methods as tools. Any public method (other than <code>reset</code>) with a docstring is automatically discovered and exposed as a callable tool. Here, the <code>guess</code> method lets the model submit a Wordle guess and receive feedback.</p> <p>For this example, we connect to the hosted environment at <a href="https://huggingface.co/spaces/openenv/wordle" rel="nofollow">openenv/wordle</a>.
For production use, we recommend duplicating the Space to your own account or running it locally via Docker, as the hosted versions have limited concurrency.</p> <p>For more information, refer to the <a href="https://huggingface.co/docs/trl/main/en/openenv" rel="nofollow">TRL-OpenEnv documentation</a>.</p> <!> <!> <p>The reward function receives the list of environment instances after each episode completes. Since the <code>WordleEnv</code> tracks its own reward (updated after each <code>guess</code> call), we simply read it out.</p> <p>This is much simpler than defining multiple reward functions manually — the environment already knows the game outcome.</p> <!> <!> <p>We create a dataset with repeated prompts to control the number of training episodes.
Each entry triggers one rollout episode during training. The prompt is formatted as a chat message.</p> <!> <!> <p>Next, we define the <strong>GRPOConfig</strong>, which controls all key training parameters.
This configuration specifies how the model interacts with vLLM, manages memory, and logs results.</p> <p>Note the <code>chat_template_kwargs=&#123;"enable_thinking": False&#125;</code> parameter — this disables Qwen3’s thinking mode so the model responds directly with tool calls instead of generating internal reasoning tokens first.</p> <!> <!> <p>Now we initialize the <code>GRPOTrainer</code> with <code>environment_factory=WordleEnv</code>.</p> <p>This tells the trainer to automatically handle the entire interaction loop:</p> <ul><li>It creates a <code>WordleEnv</code> instance for each episode.</li> <li>It generates model completions, parses tool calls (like <code>guess</code>), and steps through the environment.</li> <li>It collects rewards and manages the <code>tool_mask</code> (which tokens are model-generated vs environment-generated) automatically.</li></ul> <p>No need to write a custom <code>rollout_func</code> or manage tokenization manually.</p> <!> <p>Show memory stats before training</p> <!> <p>And train!</p> <!> <p>Show memory stats after training</p> <!> <!> <!> <!> <p>Now let’s test our fine-tuned model by loading it and playing a game of Wordle.
We use the same <code>WordleEnv</code> class to interact with the environment, and generate model responses with standard Transformers inference.</p> <!> <!> <p>Let’s play the game!</p> <!> <!> <p></p>`,1);function ll(X,Q){F(Q,!1),z(()=>{new URLSearchParams(window.location.search).get("fw")}),k();var n=D();E("tegcg5",G=>{var v=K();S(v,"content",L),_(G,v)});var e=l(H(n),2);Y(e,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var t=l(e,2);a(t,{title:"OpenEnv Wordle with GRPO using TRL",local:"openenv-wordle-with-grpo-using-trl",headingTag:"h1"});var M=l(t,24);a(M,{title:"Install dependencies",local:"install-dependencies",headingTag:"h2"});var o=l(M,4);s(o,{code:"cGlwJTIwaW5zdGFsbCUyMC1VcSUyMHRybCU1QnZsbG0lNUQlMjBnaXQlMkJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGc3BhY2VzJTJGb3BlbmVudiUyRndvcmRsZSUyMHRyYWNraW8=",highlighted:"pip install -Uq trl[vllm] git+https://huggingface.co/spaces/openenv/wordle trackio",lang:"bash",wrap:!1});var r=l(o,2);a(r,{title:"Log in to Hugging Face",local:"log-in-to-hugging-face",headingTag:"h3"});var y=l(r,4);s(y,{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,lang:"python",wrap:!1});var i=l(y,2);a(i,{title:"Define the system prompt",local:"define-the-system-prompt",headingTag:"h2"});var p=l(i,4);s(p,{code:"cHJvbXB0JTIwJTNEJTIwJTIyJTIyJTIyWW91JTIwYXJlJTIwYW4lMjBleHBlcnQlMjBXb3JkbGUlMjBzb2x2ZXIlMjB3aXRoJTIwZGVlcCUyMGtub3dsZWRnZSUyMG9mJTIwRW5nbGlzaCUyMHZvY2FidWxhcnklMkMlMjBsZXR0ZXIlMjBmcmVxdWVuY3klMjBwYXR0ZXJucyUyQyUyMGFuZCUyMG9wdGltYWwlMjBndWVzc2luZyUyMHN0cmF0ZWdpZXMuJTBBJTBBRm9sbG93JTIwdGhlc2UlMjBydWxlcyUyMHRvJTIwcGxheSUyMFdvcmRsZSUzQSUwQSUwQTEuJTIwVGhlJTIwdGFyZ2V0JTIwaXMlMjBhJTIwNS1sZXR0ZXIlMjBFbmdsaXNoJTIwd29yZCUwQTIuJTIwWW91JTIwaGF2ZSUyMDYlMjBhdHRlbXB0cyUyMHRvJTIwZ3Vlc3MlMjB0aGUlMjBjb3JyZWN0JTIwd29yZCUwQTMuJTIwQWZ0ZXIlMjBlYWNoJTIwZ3Vlc3MlMkMlMjB5b3UlMjByZWNlaXZlJTIwY29sb3ItY29kZWQlMjBmZWVkYmFjayUzQSUwQSUyMCUyMCUyMC0lMjBHUkVFTiUyMChHKSUzQSUyMExldHRlciUyMGlzJTIwY29ycmVjdCUyMGFuZCUyMGluJTIwdGhlJTIwY29ycmVjdCUyMHBvc2l0aW9uJTBBJTIwJTIwJTIwLSUyMFlFTExPVyUyMChZKSUzQSUyMExldHRlciUyMGlzJTIwaW4lMjB0aGUlMjB3b3JkJTIwYnV0JTIwaW4lMjB0aGUlMjB3cm9uZyUyMHBvc2l0aW9uJTBBJTIwJTIwJTIwLSUyMEdSQVklMjAoWCklM0ElMjBMZXR0ZXIlMjBpcyUyMG5vdCUyMGluJTIwdGhlJTIwd29yZCUyMGF0JTIwYWxsJTBBNC4lMjBBbGwlMjBndWVzc2VzJTIwbXVzdCUyMGJlJTIwdmFsaWQlMjA1LWxldHRlciUyMEVuZ2xpc2glMjB3b3JkcyUwQTUuJTIwWW91JTIwY2Fubm90JTIwcmV1c2UlMjBhJTIwd29yZCUyMHlvdSd2ZSUyMGFscmVhZHklMjBndWVzc2VkJTBBNi4lMjBVc2UlMjB0aGUlMjB0b29sJTIwJTYwZ3Vlc3MlNjAlMjB0byUyMG1ha2UlMjBhJTIwZ3Vlc3MuJTBBJTIyJTIyJTIy",highlighted:`prompt = <span class="hljs-string">&quot;&quot;&quot;You are an expert Wordle solver with deep knowledge of English vocabulary, letter frequency patterns, and optimal guessing strategies.

Follow these rules to play Wordle:

1. The target is a 5-letter English word
2. You have 6 attempts to guess the correct word
3. After each guess, you receive color-coded feedback:
   - GREEN (G): Letter is correct and in the correct position
   - YELLOW (Y): Letter is in the word but in the wrong position
   - GRAY (X): Letter is not in the word at all
4. All guesses must be valid 5-letter English words
5. You cannot reuse a word you&#x27;ve already guessed
6. Use the tool \`guess\` to make a guess.
&quot;&quot;&quot;</span>`,lang:"python",wrap:!1});var c=l(p,2);a(c,{title:"Define the environment",local:"define-the-environment",headingTag:"h2"});var J=l(c,14);s(J,{code:"ZnJvbSUyMHRleHRhcmVuYV9lbnYlMjBpbXBvcnQlMjBUZXh0QXJlbmFBY3Rpb24lMkMlMjBUZXh0QXJlbmFFbnYlMEElMEElMEFjbGFzcyUyMFdvcmRsZUVudiUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5jbGllbnQlMjAlM0QlMjBUZXh0QXJlbmFFbnYoYmFzZV91cmwlM0QlMjJodHRwcyUzQSUyRiUyRm9wZW5lbnYtd29yZGxlLmhmLnNwYWNlJTIyKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMHJlc2V0KHNlbGYlMkMlMjAqKmt3YXJncyklMjAtJTNFJTIwTm9uZSUyMCU3QyUyMHN0ciUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlc3VsdCUyMCUzRCUyMHNlbGYuY2xpZW50LnJlc2V0KCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBUaGUlMjBnYW1lJTIwcmV0dXJucyUyMGN1bXVsYXRpdmUlMjBmZWVkYmFjayUyMGVhY2glMjB0dXJuJTIwKG5ldyUyMHRleHQlMjBhcHBlbmRlZCUyMGF0JTIwdGhlJTIwZW5kKSUyQyUyMHNvJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwd2UlMjBzdG9yZSUyMHRoZSUyMHByZXZpb3VzJTIwZnVsbCUyMHJlc3BvbnNlJTIwYW5kJTIwc2xpY2UlMjBvdXQlMjBvbmx5JTIwdGhlJTIwbmV3bHklMjBhcHBlbmRlZCUyMHBhcnQuJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5fbGFzdF9mdWxsX2ZlZWRiYWNrJTIwJTNEJTIwcmVzdWx0Lm9ic2VydmF0aW9uLm1lc3NhZ2VzJTVCMCU1RC5jb250ZW50JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5yZXdhcmQlMjAlM0QlMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmRvbmUlMjAlM0QlMjBGYWxzZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMHNlbGYuX2xhc3RfZnVsbF9mZWVkYmFjayUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGd1ZXNzKHNlbGYlMkMlMjBndWVzcyUzQSUyMHN0ciklMjAtJTNFJTIwc3RyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTWFrZSUyMGElMjBndWVzcyUyMGluJTIwdGhlJTIwV29yZGxlJTIwZW52aXJvbm1lbnQuJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwQXJncyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGd1ZXNzJTNBJTIwVGhlJTIwZ3Vlc3NlZCUyMHdvcmQlMkMlMjBmb3JtYXR0ZWQlMjBhcyUyMCclNUJhYmNkZSU1RCclMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBSZXR1cm5zJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwVGhlJTIwZmVlZGJhY2slMjBtZXNzYWdlJTIwZnJvbSUyMHRoZSUyMGVudmlyb25tZW50LiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwc2VsZi5kb25lJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmFpc2UlMjBWYWx1ZUVycm9yKCUyMkdhbWUlMjBvdmVyLiUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXN1bHQlMjAlM0QlMjBzZWxmLmNsaWVudC5zdGVwKFRleHRBcmVuYUFjdGlvbihtZXNzYWdlJTNEZ3Vlc3MpKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMF9mdWxsX2ZlZWRiYWNrJTIwJTNEJTIwcmVzdWx0Lm9ic2VydmF0aW9uLm1lc3NhZ2VzJTVCMCU1RC5jb250ZW50JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwSnVzdCUyMHRha2UlMjB0aGUlMjBuZXclMjBmZWVkYmFjayUyMHNpbmNlJTIwdGhlJTIwbGFzdCUyMGd1ZXNzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmVlZGJhY2slMjAlM0QlMjBfZnVsbF9mZWVkYmFjayU1QmxlbihzZWxmLl9sYXN0X2Z1bGxfZmVlZGJhY2spJTNBJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5fbGFzdF9mdWxsX2ZlZWRiYWNrJTIwJTNEJTIwX2Z1bGxfZmVlZGJhY2slMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBQZW5hbGl6ZSUyMGludmFsaWQlMjBtb3ZlcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwJTIyWW91JTIwYXR0ZW1wdGVkJTIwYW4lMjBpbnZhbGlkJTIwbW92ZSUyMiUyMGluJTIwZmVlZGJhY2slM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLnJld2FyZCUyMCUzRCUyMDAuMCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLnJld2FyZCUyMCUzRCUyMHJlc3VsdC5yZXdhcmQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmRvbmUlMjAlM0QlMjByZXN1bHQuZG9uZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMGZlZWRiYWNr",highlighted:`<span class="hljs-keyword">from</span> textarena_env <span class="hljs-keyword">import</span> TextArenaAction, TextArenaEnv


<span class="hljs-keyword">class</span> <span class="hljs-title class_">WordleEnv</span>:
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-variable language_">self</span>.client = TextArenaEnv(base_url=<span class="hljs-string">&quot;https://openenv-wordle.hf.space&quot;</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, **kwargs</span>) -&gt; <span class="hljs-literal">None</span> | <span class="hljs-built_in">str</span>:
        result = <span class="hljs-variable language_">self</span>.client.reset()
        <span class="hljs-comment"># The game returns cumulative feedback each turn (new text appended at the end), so</span>
        <span class="hljs-comment"># we store the previous full response and slice out only the newly appended part.</span>
        <span class="hljs-variable language_">self</span>._last_full_feedback = result.observation.messages[<span class="hljs-number">0</span>].content
        <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span>
        <span class="hljs-variable language_">self</span>.done = <span class="hljs-literal">False</span>
        <span class="hljs-keyword">return</span> <span class="hljs-variable language_">self</span>._last_full_feedback

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">guess</span>(<span class="hljs-params">self, guess: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span>:
        <span class="hljs-string">&quot;&quot;&quot;
        Make a guess in the Wordle environment.

        Args:
            guess: The guessed word, formatted as &#x27;[abcde]&#x27;

        Returns:
            The feedback message from the environment.
        &quot;&quot;&quot;</span>
        <span class="hljs-keyword">if</span> <span class="hljs-variable language_">self</span>.done:
            <span class="hljs-keyword">raise</span> ValueError(<span class="hljs-string">&quot;Game over.&quot;</span>)
        result = <span class="hljs-variable language_">self</span>.client.step(TextArenaAction(message=guess))
        _full_feedback = result.observation.messages[<span class="hljs-number">0</span>].content
        <span class="hljs-comment"># Just take the new feedback since the last guess</span>
        feedback = _full_feedback[<span class="hljs-built_in">len</span>(<span class="hljs-variable language_">self</span>._last_full_feedback):]
        <span class="hljs-variable language_">self</span>._last_full_feedback = _full_feedback
        <span class="hljs-comment"># Penalize invalid moves</span>
        <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;You attempted an invalid move&quot;</span> <span class="hljs-keyword">in</span> feedback:
            <span class="hljs-variable language_">self</span>.reward = <span class="hljs-number">0.0</span>
        <span class="hljs-keyword">else</span>:
            <span class="hljs-variable language_">self</span>.reward = result.reward
        <span class="hljs-variable language_">self</span>.done = result.done
        <span class="hljs-keyword">return</span> feedback`,lang:"python",wrap:!1});var d=l(J,2);a(d,{title:"Define the reward function",local:"define-the-reward-function",headingTag:"h2"});var w=l(d,6);s(w,{code:"ZGVmJTIwcmV3YXJkX2Z1bmMoZW52aXJvbm1lbnRzJTJDJTIwKiprd2FyZ3MpJTIwLSUzRSUyMGxpc3QlNUJmbG9hdCU1RCUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QmVudi5yZXdhcmQlMjBmb3IlMjBlbnYlMjBpbiUyMGVudmlyb25tZW50cyU1RA==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">environments, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
    <span class="hljs-keyword">return</span> [env.reward <span class="hljs-keyword">for</span> env <span class="hljs-keyword">in</span> environments]`,lang:"python",wrap:!1});var U=l(w,2);a(U,{title:"Create the dataset",local:"create-the-dataset",headingTag:"h2"});var T=l(U,4);s(T,{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQSUwQWRhdGFzZXQlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglN0IlMjJwcm9tcHQlMjIlM0ElMjAlNUIlNUIlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjBwcm9tcHQlN0QlNUQlMjBmb3IlMjBfJTIwaW4lMjByYW5nZSgzMDAwKSU1RCU3RCk=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

dataset = Dataset.from_dict({<span class="hljs-string">&quot;prompt&quot;</span>: [[{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt}] <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3000</span>)]})`,lang:"python",wrap:!1});var j=l(T,2);a(j,{title:"Set GRPO Config",local:"set-grpo-config",headingTag:"h2"});var h=l(j,6);s(h,{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMEElMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyUXdlbiUyRlF3ZW4zLTEuN0IlMjIlMEFvdXRwdXRfZGlyJTIwJTNEJTIwJTIyd29yZGxlLWdycG8tUXdlbjMtMS43QiUyMiUwQSUwQWdycG9fY29uZmlnJTIwJTNEJTIwR1JQT0NvbmZpZyglMEElMjAlMjAlMjAlMjAlMjMlMjBUcmFpbmluZyUyMHNjaGVkdWxlJTIwJTJGJTIwb3B0aW1pemF0aW9uJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX2Vwb2NocyUzRDElMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMWUtNiUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDY0JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNEMSUyQyUwQSUyMCUyMCUyMCUyMHdhcm11cF9zdGVwcyUzRDEwJTJDJTBBJTIwJTIwJTIwJTIwb3B0aW0lM0QlMjJhZGFtd190b3JjaCUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9ncmFkX25vcm0lM0QxLjAlMkMlMEElMEElMjAlMjAlMjAlMjAlMjMlMjBHUlBPJTIwY29uZmlndXJhdGlvbiUwQSUyMCUyMCUyMCUyMG51bV9nZW5lcmF0aW9ucyUzRDIlMkMlMEElMjAlMjAlMjAlMjBtYXhfY29tcGxldGlvbl9sZW5ndGglM0QxMDI0JTJDJTBBJTIwJTIwJTIwJTIwbG9nX2NvbXBsZXRpb25zJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMG51bV9jb21wbGV0aW9uc190b19wcmludCUzRDIlMkMlMEElMjAlMjAlMjAlMjBjaGF0X3RlbXBsYXRlX2t3YXJncyUzRCU3QiUyMmVuYWJsZV90aGlua2luZyUyMiUzQSUyMEZhbHNlJTdEJTJDJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwdkxMTSUyMGNvbmZpZ3VyYXRpb24lMEElMjAlMjAlMjAlMjB1c2VfdmxsbSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjB2bGxtX21vZGUlM0QlMjJjb2xvY2F0ZSUyMiUyQyUwQSUyMCUyMCUyMCUyMHZsbG1fZ3B1X21lbW9yeV91dGlsaXphdGlvbiUzRDAuMTUlMkMlMEElMjAlMjAlMjAlMjB2bGxtX21heF9tb2RlbF9sZW5ndGglM0QzMDcyJTJDJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwTG9nZ2luZyUyMCUyRiUyMHJlcG9ydGluZyUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0RvdXRwdXRfZGlyJTJDJTBBJTIwJTIwJTIwJTIwcmVwb3J0X3RvJTNEJTIydHJhY2tpbyUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYWNraW9fc3BhY2VfaWQlM0RvdXRwdXRfZGlyJTJDJTBBJTIwJTIwJTIwJTIwbG9nZ2luZ19zdGVwcyUzRDElMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0ZXBzJTNEMTAlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3RvdGFsX2xpbWl0JTNEMSUyQyUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyME1lbW9yeSUyMG9wdGltaXphdGlvbiUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2NoZWNrcG9pbnRpbmclM0RUcnVlJTJDJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwSHViJTIwaW50ZWdyYXRpb24lMEElMjAlMjAlMjAlMjBwdXNoX3RvX2h1YiUzRFRydWUlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig

model_name = <span class="hljs-string">&quot;Qwen/Qwen3-1.7B&quot;</span>
output_dir = <span class="hljs-string">&quot;wordle-grpo-Qwen3-1.7B&quot;</span>

grpo_config = GRPOConfig(
    <span class="hljs-comment"># Training schedule / optimization</span>
    num_train_epochs=<span class="hljs-number">1</span>,
    learning_rate=<span class="hljs-number">1e-6</span>,
    gradient_accumulation_steps=<span class="hljs-number">64</span>,
    per_device_train_batch_size=<span class="hljs-number">1</span>,
    warmup_steps=<span class="hljs-number">10</span>,
    optim=<span class="hljs-string">&quot;adamw_torch&quot;</span>,
    max_grad_norm=<span class="hljs-number">1.0</span>,

    <span class="hljs-comment"># GRPO configuration</span>
    num_generations=<span class="hljs-number">2</span>,
    max_completion_length=<span class="hljs-number">1024</span>,
    log_completions=<span class="hljs-literal">True</span>,
    num_completions_to_print=<span class="hljs-number">2</span>,
    chat_template_kwargs={<span class="hljs-string">&quot;enable_thinking&quot;</span>: <span class="hljs-literal">False</span>},

    <span class="hljs-comment"># vLLM configuration</span>
    use_vllm=<span class="hljs-literal">True</span>,
    vllm_mode=<span class="hljs-string">&quot;colocate&quot;</span>,
    vllm_gpu_memory_utilization=<span class="hljs-number">0.15</span>,
    vllm_max_model_length=<span class="hljs-number">3072</span>,

    <span class="hljs-comment"># Logging / reporting</span>
    output_dir=output_dir,
    report_to=<span class="hljs-string">&quot;trackio&quot;</span>,
    trackio_space_id=output_dir,
    logging_steps=<span class="hljs-number">1</span>,
    save_steps=<span class="hljs-number">10</span>,
    save_total_limit=<span class="hljs-number">1</span>,

    <span class="hljs-comment"># Memory optimization</span>
    gradient_checkpointing=<span class="hljs-literal">True</span>,

    <span class="hljs-comment"># Hub integration</span>
    push_to_hub=<span class="hljs-literal">True</span>,
)`,lang:"python",wrap:!1});var m=l(h,2);a(m,{title:"Create the GRPOTrainer and start training",local:"create-the-grpotrainer-and-start-training",headingTag:"h2"});var u=l(m,10);s(u,{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTBBJTBBdHJhaW5lciUyMCUzRCUyMEdSUE9UcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWxfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRHJld2FyZF9mdW5jJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEZ3Jwb19jb25maWclMkMlMEElMjAlMjAlMjAlMjBlbnZpcm9ubWVudF9mYWN0b3J5JTNEV29yZGxlRW52JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer

trainer = GRPOTrainer(
    model=model_name,
    reward_funcs=reward_func,
    train_dataset=dataset,
    args=grpo_config,
    environment_factory=WordleEnv,
)`,lang:"python",wrap:!1});var I=l(u,4);s(I,{code:"aW1wb3J0JTIwdG9yY2glMEElMEFncHVfc3RhdHMlMjAlM0QlMjB0b3JjaC5jdWRhLmdldF9kZXZpY2VfcHJvcGVydGllcygwKSUwQXN0YXJ0X2dwdV9tZW1vcnklMjAlM0QlMjByb3VuZCh0b3JjaC5jdWRhLm1heF9tZW1vcnlfcmVzZXJ2ZWQoKSUyMCUyRiUyMDEwMjQlMjAlMkYlMjAxMDI0JTIwJTJGJTIwMTAyNCUyQyUyMDMpJTBBbWF4X21lbW9yeSUyMCUzRCUyMHJvdW5kKGdwdV9zdGF0cy50b3RhbF9tZW1vcnklMjAlMkYlMjAxMDI0JTIwJTJGJTIwMTAyNCUyMCUyRiUyMDEwMjQlMkMlMjAzKSUwQSUwQXByaW50KGYlMjJHUFUlMjAlM0QlMjAlN0JncHVfc3RhdHMubmFtZSU3RC4lMjBNYXglMjBtZW1vcnklMjAlM0QlMjAlN0JtYXhfbWVtb3J5JTdEJTIwR0IuJTIyKSUwQXByaW50KGYlMjIlN0JzdGFydF9ncHVfbWVtb3J5JTdEJTIwR0IlMjBvZiUyMG1lbW9yeSUyMHJlc2VydmVkLiUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch

gpu_stats = torch.cuda.get_device_properties(<span class="hljs-number">0</span>)
start_gpu_memory = <span class="hljs-built_in">round</span>(torch.cuda.max_memory_reserved() / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span>, <span class="hljs-number">3</span>)
max_memory = <span class="hljs-built_in">round</span>(gpu_stats.total_memory / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span>, <span class="hljs-number">3</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;GPU = <span class="hljs-subst">{gpu_stats.name}</span>. Max memory = <span class="hljs-subst">{max_memory}</span> GB.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{start_gpu_memory}</span> GB of memory reserved.&quot;</span>)`,lang:"python",wrap:!1});var b=l(I,4);s(b,{code:"dHJhaW5lcl9zdGF0cyUyMCUzRCUyMHRyYWluZXIudHJhaW4oKQ==",highlighted:"trainer_stats = trainer.train()",lang:"python",wrap:!1});var g=l(b,4);s(g,{code:"dXNlZF9tZW1vcnklMjAlM0QlMjByb3VuZCh0b3JjaC5jdWRhLm1heF9tZW1vcnlfcmVzZXJ2ZWQoKSUyMCUyRiUyMDEwMjQlMjAlMkYlMjAxMDI0JTIwJTJGJTIwMTAyNCUyQyUyMDMpJTBBdXNlZF9tZW1vcnlfZm9yX3RyYWluaW5nJTIwJTNEJTIwcm91bmQodXNlZF9tZW1vcnklMjAtJTIwc3RhcnRfZ3B1X21lbW9yeSUyQyUyMDMpJTBBdXNlZF9wZXJjZW50YWdlJTIwJTNEJTIwcm91bmQodXNlZF9tZW1vcnklMjAlMkYlMjBtYXhfbWVtb3J5JTIwKiUyMDEwMCUyQyUyMDMpJTBBdHJhaW5pbmdfbWVtb3J5X3BlcmNlbnRhZ2UlMjAlM0QlMjByb3VuZCh1c2VkX21lbW9yeV9mb3JfdHJhaW5pbmclMjAlMkYlMjBtYXhfbWVtb3J5JTIwKiUyMDEwMCUyQyUyMDMpJTBBJTBBcHJpbnQoZiUyMiU3QnRyYWluZXJfc3RhdHMubWV0cmljcyU1Qid0cmFpbl9ydW50aW1lJyU1RCU3RCUyMHNlY29uZHMlMjB1c2VkJTIwZm9yJTIwdHJhaW5pbmcuJTIyKSUwQXByaW50KGYlMjIlN0Jyb3VuZCh0cmFpbmVyX3N0YXRzLm1ldHJpY3MlNUIndHJhaW5fcnVudGltZSclNUQlMkY2MCUyQyUyMDIpJTdEJTIwbWludXRlcyUyMHVzZWQlMjBmb3IlMjB0cmFpbmluZy4lMjIpJTBBcHJpbnQoZiUyMlBlYWslMjByZXNlcnZlZCUyMG1lbW9yeSUyMCUzRCUyMCU3QnVzZWRfbWVtb3J5JTdEJTIwR0IuJTIyKSUwQXByaW50KGYlMjJQZWFrJTIwcmVzZXJ2ZWQlMjBtZW1vcnklMjBmb3IlMjB0cmFpbmluZyUyMCUzRCUyMCU3QnVzZWRfbWVtb3J5X2Zvcl90cmFpbmluZyU3RCUyMEdCLiUyMiklMEFwcmludChmJTIyUGVhayUyMHJlc2VydmVkJTIwbWVtb3J5JTIwJTI1JTIwb2YlMjBtYXglMjBtZW1vcnklMjAlM0QlMjAlN0J1c2VkX3BlcmNlbnRhZ2UlN0QlMjAlMjUuJTIyKSUwQXByaW50KGYlMjJQZWFrJTIwcmVzZXJ2ZWQlMjBtZW1vcnklMjBmb3IlMjB0cmFpbmluZyUyMCUyNSUyMG9mJTIwbWF4JTIwbWVtb3J5JTIwJTNEJTIwJTdCdHJhaW5pbmdfbWVtb3J5X3BlcmNlbnRhZ2UlN0QlMjAlMjUuJTIyKQ==",highlighted:`used_memory = <span class="hljs-built_in">round</span>(torch.cuda.max_memory_reserved() / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span>, <span class="hljs-number">3</span>)
used_memory_for_training = <span class="hljs-built_in">round</span>(used_memory - start_gpu_memory, <span class="hljs-number">3</span>)
used_percentage = <span class="hljs-built_in">round</span>(used_memory / max_memory * <span class="hljs-number">100</span>, <span class="hljs-number">3</span>)
training_memory_percentage = <span class="hljs-built_in">round</span>(used_memory_for_training / max_memory * <span class="hljs-number">100</span>, <span class="hljs-number">3</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{trainer_stats.metrics[<span class="hljs-string">&#x27;train_runtime&#x27;</span>]}</span> seconds used for training.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{<span class="hljs-built_in">round</span>(trainer_stats.metrics[<span class="hljs-string">&#x27;train_runtime&#x27;</span>]/<span class="hljs-number">60</span>, <span class="hljs-number">2</span>)}</span> minutes used for training.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Peak reserved memory = <span class="hljs-subst">{used_memory}</span> GB.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Peak reserved memory for training = <span class="hljs-subst">{used_memory_for_training}</span> GB.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Peak reserved memory % of max memory = <span class="hljs-subst">{used_percentage}</span> %.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Peak reserved memory for training % of max memory = <span class="hljs-subst">{training_memory_percentage}</span> %.&quot;</span>)`,lang:"python",wrap:!1});var C=l(g,2);a(C,{title:"Save and push to Hub",local:"save-and-push-to-hub",headingTag:"h2"});var Z=l(C,2);s(Z,{code:"dHJhaW5lci5zYXZlX21vZGVsKG91dHB1dF9kaXIpJTBBdHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:`trainer.save_model(output_dir)
trainer.push_to_hub()`,lang:"python",wrap:!1});var B=l(Z,2);a(B,{title:"Load the fine-tuned model and run inference",local:"load-the-fine-tuned-model-and-run-inference",headingTag:"h2"});var A=l(B,4);s(A,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJzZXJnaW9wYW5pZWdvJTJGd29yZGxlLWdycG8tUXdlbjMtMS43QiUyMiUyMCUyMCUyMyUyMFJlcGxhY2UlMjB3aXRoJTIweW91ciUyMEhGJTIwdXNlcm5hbWUlMjBvciUyMG9yZ2FuaXphdGlvbiUwQSUwQWZpbmVfdHVuZWRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSUyQyUyMHRvcmNoX2R0eXBlJTNEJTIyZmxvYXQzMiUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;sergiopaniego/wordle-grpo-Qwen3-1.7B&quot;</span>  <span class="hljs-comment"># Replace with your HF username or organization</span>

fine_tuned_model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=<span class="hljs-string">&quot;float32&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
tokenizer = AutoTokenizer.from_pretrained(model_name)`,lang:"python",wrap:!1});var f=l(A,2);s(f,{code:"aW1wb3J0JTIwanNvbiUwQSUwQSUwQWRlZiUyMHBsYXlfd29yZGxlKG1vZGVsJTJDJTIwdG9rZW5pemVyKSUzQSUwQSUyMCUyMCUyMCUyMGVudiUyMCUzRCUyMFdvcmRsZUVudigpJTBBJTIwJTIwJTIwJTIwaW5pdGlhbF9vYnNlcnZhdGlvbiUyMCUzRCUyMGVudi5yZXNldCgpJTBBJTBBJTIwJTIwJTIwJTIwcHJpbnQoJTIySW5pdGlhbCUyMG9ic2VydmF0aW9uJTNBJTIyKSUwQSUyMCUyMCUyMCUyMHByaW50KGluaXRpYWxfb2JzZXJ2YXRpb24pJTBBJTIwJTIwJTIwJTIwcHJpbnQoKSUwQSUwQSUyMCUyMCUyMCUyMG1lc3NhZ2VzJTIwJTNEJTIwJTVCJTdCJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwcHJvbXB0JTdEJTVEJTBBJTIwJTIwJTIwJTIwaWYlMjBpbml0aWFsX29ic2VydmF0aW9uJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWVzc2FnZXMuYXBwZW5kKCU3QiUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMGluaXRpYWxfb2JzZXJ2YXRpb24lN0QpJTBBJTBBJTIwJTIwJTIwJTIwZm9yJTIwdHVybiUyMGluJTIwcmFuZ2UoNiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGVudi5kb25lJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYnJlYWslMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcm9tcHRfdGV4dCUyMCUzRCUyMHRva2VuaXplci5hcHBseV9jaGF0X3RlbXBsYXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1lc3NhZ2VzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWRkX2dlbmVyYXRpb25fcHJvbXB0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRva2VuaXplJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmFibGVfdGhpbmtpbmclM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTVCcHJvbXB0X3RleHQlNUQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqbW9kZWxfaW5wdXRzJTJDJTIwbWF4X25ld190b2tlbnMlM0Q1MTIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0X2lkcyUyMCUzRCUyMGdlbmVyYXRlZF9pZHMlNUIwJTVEJTVCbGVuKG1vZGVsX2lucHV0cy5pbnB1dF9pZHMlNUIwJTVEKSUzQSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGdlbmVyYXRlZF90ZXh0JTIwJTNEJTIwdG9rZW5pemVyLmRlY29kZShvdXRwdXRfaWRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZiUyMlR1cm4lMjAlN0J0dXJuJTIwJTJCJTIwMSU3RCUyMC0lMjBNb2RlbCUyMG91dHB1dCUzQSUyMCU3QmdlbmVyYXRlZF90ZXh0JTdEJTIyKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFRyeSUyMHRvJTIwcGFyc2UlMjB0b29sJTIwY2FsbCUyMGZyb20lMjB0aGUlMjBnZW5lcmF0ZWQlMjB0ZXh0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJ5JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwVHJ5JTIwdG8lMjBleHRyYWN0JTIwYSUyMGd1ZXNzJTIwZnJvbSUyMHRvb2wlMjBjYWxsJTIwZm9ybWF0JTIwb3IlMjBicmFja2V0JTIwZm9ybWF0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjAlMjJndWVzcyUyMiUyMGluJTIwZ2VuZXJhdGVkX3RleHQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBQYXJzZSUyMEpTT04lMjB0b29sJTIwY2FsbCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXJ0JTIwJTNEJTIwZ2VuZXJhdGVkX3RleHQuaW5kZXgoJTIyJTdCJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVuZCUyMCUzRCUyMGdlbmVyYXRlZF90ZXh0LnJpbmRleCglMjIlN0QlMjIpJTIwJTJCJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFyZ3MlMjAlM0QlMjBqc29uLmxvYWRzKGdlbmVyYXRlZF90ZXh0JTVCc3RhcnQlM0FlbmQlNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjAlMjJhcmd1bWVudHMlMjIlMjBpbiUyMGFyZ3MlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhcmdzJTIwJTNEJTIwYXJncyU1QiUyMmFyZ3VtZW50cyUyMiU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGd1ZXNzX3dvcmQlMjAlM0QlMjBhcmdzLmdldCglMjJndWVzcyUyMiUyQyUyMCUyMiUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbHNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwRmFsbGJhY2slM0ElMjBleHRyYWN0JTIwZnJvbSUyMGJyYWNrZXRzJTIwJTVCd29yZCU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGltcG9ydCUyMHJlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWF0Y2glMjAlM0QlMjByZS5zZWFyY2gociUyMiU1QyU1QiglNUJhLXpBLVolNUQlN0I1JTdEKSU1QyU1RCUyMiUyQyUyMGdlbmVyYXRlZF90ZXh0KSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGd1ZXNzX3dvcmQlMjAlM0QlMjBtYXRjaC5ncm91cCgxKSUyMGlmJTIwbWF0Y2glMjBlbHNlJTIwZ2VuZXJhdGVkX3RleHQuc3RyaXAoKSU1QiUzQTUlNUQlMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWVkYmFjayUyMCUzRCUyMGVudi5ndWVzcyhmJTIyJTVCJTdCZ3Vlc3Nfd29yZCU3RCU1RCUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludChmJTIyJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR3Vlc3MlM0ElMjAlN0JndWVzc193b3JkJTdEJTIwJTdDJTIwUmV3YXJkJTNBJTIwJTdCZW52LnJld2FyZCU3RCUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludChmJTIyJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwRmVlZGJhY2slM0ElMjAlN0JmZWVkYmFjay5zdHJpcCgpJTdEJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtZXNzYWdlcy5hcHBlbmQoJTdCJTIycm9sZSUyMiUzQSUyMCUyMmFzc2lzdGFudCUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjBnZW5lcmF0ZWRfdGV4dCU3RCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtZXNzYWdlcy5hcHBlbmQoJTdCJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwZmVlZGJhY2slN0QpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXhjZXB0JTIwRXhjZXB0aW9uJTIwYXMlMjBlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEVycm9yJTNBJTIwJTdCZSU3RCUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBicmVhayUwQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJHYW1lJTIwZmluaXNoZWQhJTIwRmluYWwlMjByZXdhcmQlM0ElMjAlN0JlbnYucmV3YXJkJTdEJTIyKSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJEb25lJTNBJTIwJTdCZW52LmRvbmUlN0QlMjIp",highlighted:`<span class="hljs-keyword">import</span> json


<span class="hljs-keyword">def</span> <span class="hljs-title function_">play_wordle</span>(<span class="hljs-params">model, tokenizer</span>):
    env = WordleEnv()
    initial_observation = env.reset()

    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Initial observation:&quot;</span>)
    <span class="hljs-built_in">print</span>(initial_observation)
    <span class="hljs-built_in">print</span>()

    messages = [{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt}]
    <span class="hljs-keyword">if</span> initial_observation:
        messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: initial_observation})

    <span class="hljs-keyword">for</span> turn <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">6</span>):
        <span class="hljs-keyword">if</span> env.done:
            <span class="hljs-keyword">break</span>

        prompt_text = tokenizer.apply_chat_template(
            messages,
            add_generation_prompt=<span class="hljs-literal">True</span>,
            tokenize=<span class="hljs-literal">False</span>,
            enable_thinking=<span class="hljs-literal">False</span>,
        )
        model_inputs = tokenizer([prompt_text], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
        generated_ids = model.generate(**model_inputs, max_new_tokens=<span class="hljs-number">512</span>)
        output_ids = generated_ids[<span class="hljs-number">0</span>][<span class="hljs-built_in">len</span>(model_inputs.input_ids[<span class="hljs-number">0</span>]):]
        generated_text = tokenizer.decode(output_ids, skip_special_tokens=<span class="hljs-literal">True</span>)

        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Turn <span class="hljs-subst">{turn + <span class="hljs-number">1</span>}</span> - Model output: <span class="hljs-subst">{generated_text}</span>&quot;</span>)

        <span class="hljs-comment"># Try to parse tool call from the generated text</span>
        <span class="hljs-keyword">try</span>:
            <span class="hljs-comment"># Try to extract a guess from tool call format or bracket format</span>
            <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;guess&quot;</span> <span class="hljs-keyword">in</span> generated_text:
                <span class="hljs-comment"># Parse JSON tool call</span>
                start = generated_text.index(<span class="hljs-string">&quot;{&quot;</span>)
                end = generated_text.rindex(<span class="hljs-string">&quot;}&quot;</span>) + <span class="hljs-number">1</span>
                args = json.loads(generated_text[start:end])
                <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;arguments&quot;</span> <span class="hljs-keyword">in</span> args:
                    args = args[<span class="hljs-string">&quot;arguments&quot;</span>]
                guess_word = args.get(<span class="hljs-string">&quot;guess&quot;</span>, <span class="hljs-string">&quot;&quot;</span>)
            <span class="hljs-keyword">else</span>:
                <span class="hljs-comment"># Fallback: extract from brackets [word]</span>
                <span class="hljs-keyword">import</span> re
                <span class="hljs-keyword">match</span> = re.search(<span class="hljs-string">r&quot;\\[([a-zA-Z]{5})\\]&quot;</span>, generated_text)
                guess_word = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">1</span>) <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">else</span> generated_text.strip()[:<span class="hljs-number">5</span>]

            feedback = env.guess(<span class="hljs-string">f&quot;[<span class="hljs-subst">{guess_word}</span>]&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;         Guess: <span class="hljs-subst">{guess_word}</span> | Reward: <span class="hljs-subst">{env.reward}</span>&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;         Feedback: <span class="hljs-subst">{feedback.strip()}</span>&quot;</span>)
            <span class="hljs-built_in">print</span>()

            messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: generated_text})
            messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: feedback})
        <span class="hljs-keyword">except</span> Exception <span class="hljs-keyword">as</span> e:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;         Error: <span class="hljs-subst">{e}</span>&quot;</span>)
            <span class="hljs-keyword">break</span>

    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Game finished! Final reward: <span class="hljs-subst">{env.reward}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Done: <span class="hljs-subst">{env.done}</span>&quot;</span>)`,lang:"python",wrap:!1});var W=l(f,4);s(W,{code:"cGxheV93b3JkbGUoZmluZV90dW5lZF9tb2RlbCUyQyUyMHRva2VuaXplcik=",highlighted:"play_wordle(fine_tuned_model, tokenizer)",lang:"python",wrap:!1});var V=l(W,2);N(V,{source:"https://github.com/huggingface/openenv/blob/main/docs/source/tutorials/wordle-grpo.md"}),q(2),_(X,n),x()}export{ll as component};