openenv/pr_797/en/_app/immutable/nodes/64.be63f736.js

import{s as Is,n as bs,o as Cs}from"../chunks/scheduler.2b22cead.js";import{S as fs,i as gs,e as M,s as n,c as y,h as Zs,a as i,d as s,b as a,f as us,g as r,j as p,k as hs,l as Bs,m as t,n as o,t as J,o as w,p as T}from"../chunks/index.1a0e8013.js";import{C as As,H as U,E as Ws}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.74ebbd08.js";import{C as m}from"../chunks/CodeBlock.41907c63.js";function vs($e){let c,Bl,gl,Al,j,Wl,d,vl,u,Ee='<a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/notebooks/openenv_wordle_grpo.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>',Gl,h,He='<img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/trl_banner_dark.png" alt="trl banner"/>',_l,I,Ye='With <a href="https://github.com/huggingface/trl" rel="nofollow"><strong>Transformers Reinforcement Learning (TRL)</strong></a>, you can train a model that learns to <strong>play Wordle</strong>, a word-guessing game, through interaction and reinforcement.',Rl,b,Ne='<li><a href="https://github.com/huggingface/trl" rel="nofollow">TRL GitHub Repository</a> — star us to support the project!</li> <li><a href="https://huggingface.co/docs/trl/example_overview" rel="nofollow">Official TRL Examples</a></li> <li><a href="https://huggingface.co/docs/trl/community_tutorials" rel="nofollow">Community Tutorials</a></li> <li><a href="https://github.com/huggingface/OpenEnv" rel="nofollow">OpenEnv</a></li>',kl,C,ze=`An <strong>agentic environment</strong> is a setting where a model can take actions, observe outcomes, and adjust its behavior based on feedback, similar to how humans learn from trial and error.
In this case, the agent interacts with the <strong>Wordle</strong> environment through the <a href="https://github.com/huggingface/OpenEnv" rel="nofollow"><strong>OpenEnv</strong></a> framework, which standardizes multi-agent and RL-style text environments.`,Xl,f,xe=`<a href="https://en.wikipedia.org/wiki/Wordle" rel="nofollow">Wordle</a> is a popular word puzzle where the player must guess a secret five-letter word within six tries.
After each guess, feedback indicates whether each letter is:`,Ql,g,Se="<li><strong>GREEN (G)</strong>: Correct and in the right position</li> <li><strong>YELLOW (Y)</strong>: Present but in the wrong position</li> <li><strong>GRAY (X)</strong>: Not in the word</li>",Vl,Z,Fe="This feedback loop makes Wordle a perfect environment for <strong>RL with LLMs</strong>, where the goal is to maximize the probability of guessing the correct word efficiently.",$l,B,Le=`We’ll fine-tune a model using <strong>GRPO</strong> (Group Relative Policy Optimization) via TRL.
Using <code>environment_factory</code>, the trainer automatically handles:`,El,A,qe="<li>Creating environment instances for each rollout.</li> <li>Generating model completions and parsing tool calls.</li> <li>Stepping through the environment with the model’s actions.</li> <li>Collecting rewards and managing the interaction loop.</li>",Hl,W,Ke="This means you only need to define the environment class and reward function — the trainer takes care of the rest.",Yl,v,Nl,G,Pe="We’ll start by installing <strong>TRL</strong> (with vLLM support), the <strong>OpenEnv</strong> Wordle environment, and <strong>trackio</strong> for logging.",zl,_,xl,R,Sl,k,De='Log in to your <strong>Hugging Face</strong> account to save your fine-tuned model, track your experiment results directly on the Hub or access gated models. You can find your <strong>access token</strong> on your <a href="https://huggingface.co/settings/tokens" rel="nofollow">account settings page</a>.',Fl,X,Ll,Q,ql,V,Oe="This prompt instructs the model on how to play Wordle. It includes the game rules, feedback format, and importantly, tells the model to use the <code>guess</code> tool to submit guesses. The <code>environment_factory</code> pattern uses tool calling to interact with the environment, so the model needs to know which tool to call.",Kl,$,Pl,E,Dl,H,ls="The <code>WordleEnv</code> class wraps the OpenEnv TextArena Wordle environment into the interface expected by <code>environment_factory</code>.",Ol,Y,es="When you pass <code>environment_factory=WordleEnv</code> to the trainer, it will:",le,N,ss="<li>Create a new <code>WordleEnv()</code> instance for each rollout episode.</li> <li>Call <code>reset()</code> to start a new game (returns the initial observation or <code>None</code>).</li> <li>Automatically generate model completions, parse tool calls, and invoke the corresponding methods (e.g., <code>guess(...)</code>).</li> <li>Repeat until the environment signals <code>done=True</code> or the max completion length is reached.</li>",ee,z,ts="The environment exposes its public methods as tools. Any public method (other than <code>reset</code>) with a docstring is automatically discovered and exposed as a callable tool. Here, the <code>guess</code> method lets the model submit a Wordle guess and receive feedback.",se,x,ns=`For this example, we connect to the hosted environment at <a href="https://huggingface.co/spaces/openenv/wordle" rel="nofollow">openenv/wordle</a>.
For production use, we recommend duplicating the Space to your own account or running it locally via Docker, as the hosted versions have limited concurrency.`,te,S,as='For more information, refer to the <a href="https://huggingface.co/docs/trl/main/en/openenv" rel="nofollow">TRL-OpenEnv documentation</a>.',ne,F,ae,L,Me,q,Ms="The reward function receives the list of environment instances after each episode completes. Since the <code>WordleEnv</code> tracks its own reward (updated after each <code>guess</code> call), we simply read it out.",ie,K,is="This is much simpler than defining multiple reward functions manually — the environment already knows the game outcome.",pe,P,ye,D,re,O,ps=`We create a dataset with repeated prompts to control the number of training episodes.
Each entry triggers one rollout episode during training. The prompt is formatted as a chat message.`,oe,ll,Je,el,we,sl,ys=`Next, we define the <strong>GRPOConfig</strong>, which controls all key training parameters.
This configuration specifies how the model interacts with vLLM, manages memory, and logs results.`,Te,tl,rs="Note the <code>chat_template_kwargs={&quot;enable_thinking&quot;: False}</code> parameter — this disables Qwen3’s thinking mode so the model responds directly with tool calls instead of generating internal reasoning tokens first.",me,nl,Ue,al,ce,Ml,os="Now we initialize the <code>GRPOTrainer</code> with <code>environment_factory=WordleEnv</code>.",je,il,Js="This tells the trainer to automatically handle the entire interaction loop:",de,pl,ws="<li>It creates a <code>WordleEnv</code> instance for each episode.</li> <li>It generates model completions, parses tool calls (like <code>guess</code>), and steps through the environment.</li> <li>It collects rewards and manages the <code>tool_mask</code> (which tokens are model-generated vs environment-generated) automatically.</li>",ue,yl,Ts="No need to write a custom <code>rollout_func</code> or manage tokenization manually.",he,rl,Ie,ol,ms="Show memory stats before training",be,Jl,Ce,wl,Us="And train!",fe,Tl,ge,ml,cs="Show memory stats after training",Ze,Ul,Be,cl,Ae,jl,We,dl,ve,ul,js=`Now let’s test our fine-tuned model by loading it and playing a game of Wordle.
We use the same <code>WordleEnv</code> class to interact with the environment, and generate model responses with standard Transformers inference.`,Ge,hl,_e,Il,Re,bl,ds="Let’s play the game!",ke,Cl,Xe,fl,Qe,Zl,Ve;return j=new As({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),d=new U({props:{title:"OpenEnv Wordle with GRPO using TRL",local:"openenv-wordle-with-grpo-using-trl",headingTag:"h1"}}),v=new U({props:{title:"Install dependencies",local:"install-dependencies",headingTag:"h2"}}),_=new m({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VcSUyMHRybCU1QnZsbG0lNUQlMjBnaXQlMkJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGc3BhY2VzJTJGb3BlbmVudiUyRndvcmRsZSUyMHRyYWNraW8=",highlighted:"pip install -Uq trl[vllm] git+https://huggingface.co/spaces/openenv/wordle trackio",lang:"bash",wrap:!1}}),R=new U({props:{title:"Log in to Hugging Face",local:"log-in-to-hugging-face",headingTag:"h3"}}),X=new m({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,lang:"python",wrap:!1}}),Q=new U({props:{title:"Define the system prompt",local:"define-the-system-prompt",headingTag:"h2"}}),$=new m({props:{code:"cHJvbXB0JTIwJTNEJTIwJTIyJTIyJTIyWW91JTIwYXJlJTIwYW4lMjBleHBlcnQlMjBXb3JkbGUlMjBzb2x2ZXIlMjB3aXRoJTIwZGVlcCUyMGtub3dsZWRnZSUyMG9mJTIwRW5nbGlzaCUyMHZvY2FidWxhcnklMkMlMjBsZXR0ZXIlMjBmcmVxdWVuY3klMjBwYXR0ZXJucyUyQyUyMGFuZCUyMG9wdGltYWwlMjBndWVzc2luZyUyMHN0cmF0ZWdpZXMuJTBBJTBBRm9sbG93JTIwdGhlc2UlMjBydWxlcyUyMHRvJTIwcGxheSUyMFdvcmRsZSUzQSUwQSUwQTEuJTIwVGhlJTIwdGFyZ2V0JTIwaXMlMjBhJTIwNS1sZXR0ZXIlMjBFbmdsaXNoJTIwd29yZCUwQTIuJTIwWW91JTIwaGF2ZSUyMDYlMjBhdHRlbXB0cyUyMHRvJTIwZ3Vlc3MlMjB0aGUlMjBjb3JyZWN0JTIwd29yZCUwQTMuJTIwQWZ0ZXIlMjBlYWNoJTIwZ3Vlc3MlMkMlMjB5b3UlMjByZWNlaXZlJTIwY29sb3ItY29kZWQlMjBmZWVkYmFjayUzQSUwQSUyMCUyMCUyMC0lMjBHUkVFTiUyMChHKSUzQSUyMExldHRlciUyMGlzJTIwY29ycmVjdCUyMGFuZCUyMGluJTIwdGhlJTIwY29ycmVjdCUyMHBvc2l0aW9uJTBBJTIwJTIwJTIwLSUyMFlFTExPVyUyMChZKSUzQSUyMExldHRlciUyMGlzJTIwaW4lMjB0aGUlMjB3b3JkJTIwYnV0JTIwaW4lMjB0aGUlMjB3cm9uZyUyMHBvc2l0aW9uJTBBJTIwJTIwJTIwLSUyMEdSQVklMjAoWCklM0ElMjBMZXR0ZXIlMjBpcyUyMG5vdCUyMGluJTIwdGhlJTIwd29yZCUyMGF0JTIwYWxsJTBBNC4lMjBBbGwlMjBndWVzc2VzJTIwbXVzdCUyMGJlJTIwdmFsaWQlMjA1LWxldHRlciUyMEVuZ2xpc2glMjB3b3JkcyUwQTUuJTIwWW91JTIwY2Fubm90JTIwcmV1c2UlMjBhJTIwd29yZCUyMHlvdSd2ZSUyMGFscmVhZHklMjBndWVzc2VkJTBBNi4lMjBVc2UlMjB0aGUlMjB0b29sJTIwJTYwZ3Vlc3MlNjAlMjB0byUyMG1ha2UlMjBhJTIwZ3Vlc3MuJTBBJTIyJTIyJTIy",highlighted:`prompt = <span class="hljs-string">&quot;&quot;&quot;You are an expert Wordle solver with deep knowledge of English vocabulary, letter frequency patterns, and optimal guessing strategies.

Follow these rules to play Wordle:

1. The target is a 5-letter English word
2. You have 6 attempts to guess the correct word
3. After each guess, you receive color-coded feedback:
   - GREEN (G): Letter is correct and in the correct position
   - YELLOW (Y): Letter is in the word but in the wrong position
   - GRAY (X): Letter is not in the word at all
4. All guesses must be valid 5-letter English words
5. You cannot reuse a word you&#x27;ve already guessed
6. Use the tool \`guess\` to make a guess.
&quot;&quot;&quot;</span>`,lang:"python",wrap:!1}}),E=new U({props:{title:"Define the environment",local:"define-the-environment",headingTag:"h2"}}),F=new m({props:{code:"ZnJvbSUyMHRleHRhcmVuYV9lbnYlMjBpbXBvcnQlMjBUZXh0QXJlbmFBY3Rpb24lMkMlMjBUZXh0QXJlbmFFbnYlMEElMEElMEFjbGFzcyUyMFdvcmRsZUVudiUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5jbGllbnQlMjAlM0QlMjBUZXh0QXJlbmFFbnYoYmFzZV91cmwlM0QlMjJodHRwcyUzQSUyRiUyRm9wZW5lbnYtd29yZGxlLmhmLnNwYWNlJTIyKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMHJlc2V0KHNlbGYlMkMlMjAqKmt3YXJncyklMjAtJTNFJTIwTm9uZSUyMCU3QyUyMHN0ciUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlc3VsdCUyMCUzRCUyMHNlbGYuY2xpZW50LnJlc2V0KCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBUaGUlMjBnYW1lJTIwcmV0dXJucyUyMGN1bXVsYXRpdmUlMjBmZWVkYmFjayUyMGVhY2glMjB0dXJuJTIwKG5ldyUyMHRleHQlMjBhcHBlbmRlZCUyMGF0JTIwdGhlJTIwZW5kKSUyQyUyMHNvJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwd2UlMjBzdG9yZSUyMHRoZSUyMHByZXZpb3VzJTIwZnVsbCUyMHJlc3BvbnNlJTIwYW5kJTIwc2xpY2UlMjBvdXQlMjBvbmx5JTIwdGhlJTIwbmV3bHklMjBhcHBlbmRlZCUyMHBhcnQuJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5fbGFzdF9mdWxsX2ZlZWRiYWNrJTIwJTNEJTIwcmVzdWx0Lm9ic2VydmF0aW9uLm1lc3NhZ2VzJTVCMCU1RC5jb250ZW50JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5yZXdhcmQlMjAlM0QlMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmRvbmUlMjAlM0QlMjBGYWxzZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMHNlbGYuX2xhc3RfZnVsbF9mZWVkYmFjayUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGd1ZXNzKHNlbGYlMkMlMjBndWVzcyUzQSUyMHN0ciklMjAtJTNFJTIwc3RyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTWFrZSUyMGElMjBndWVzcyUyMGluJTIwdGhlJTIwV29yZGxlJTIwZW52aXJvbm1lbnQuJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwQXJncyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGd1ZXNzJTNBJTIwVGhlJTIwZ3Vlc3NlZCUyMHdvcmQlMkMlMjBmb3JtYXR0ZWQlMjBhcyUyMCclNUJhYmNkZSU1RCclMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBSZXR1cm5zJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwVGhlJTIwZmVlZGJhY2slMjBtZXNzYWdlJTIwZnJvbSUyMHRoZSUyMGVudmlyb25tZW50LiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwc2VsZi5kb25lJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmFpc2UlMjBWYWx1ZUVycm9yKCUyMkdhbWUlMjBvdmVyLiUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXN1bHQlMjAlM0QlMjBzZWxmLmNsaWVudC5zdGVwKFRleHRBcmVuYUFjdGlvbihtZXNzYWdlJTNEZ3Vlc3MpKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMF9mdWxsX2ZlZWRiYWNrJTIwJTNEJTIwcmVzdWx0Lm9ic2VydmF0aW9uLm1lc3NhZ2VzJTVCMCU1RC5jb250ZW50JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwSnVzdCUyMHRha2UlMjB0aGUlMjBuZXclMjBmZWVkYmFjayUyMHNpbmNlJTIwdGhlJTIwbGFzdCUyMGd1ZXNzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmVlZGJhY2slMjAlM0QlMjBfZnVsbF9mZWVkYmFjayU1QmxlbihzZWxmLl9sYXN0X2Z1bGxfZmVlZGJhY2spJTNBJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5fbGFzdF9mdWxsX2ZlZWRiYWNrJTIwJTNEJTIwX2Z1bGxfZmVlZGJhY2slMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBQZW5hbGl6ZSUyMGludmFsaWQlMjBtb3ZlcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwJTIyWW91JTIwYXR0ZW1wdGVkJTIwYW4lMjBpbnZhbGlkJTIwbW92ZSUyMiUyMGluJTIwZmVlZGJhY2slM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLnJld2FyZCUyMCUzRCUyMDAuMCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLnJld2FyZCUyMCUzRCUyMHJlc3VsdC5yZXdhcmQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmRvbmUlMjAlM0QlMjByZXN1bHQuZG9uZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMGZlZWRiYWNr",highlighted:`<span class="hljs-keyword">from</span> textarena_env <span class="hljs-keyword">import</span> TextArenaAction, TextArenaEnv


<span class="hljs-keyword">class</span> <span class="hljs-title class_">WordleEnv</span>:
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        self.client = TextArenaEnv(base_url=<span class="hljs-string">&quot;https://openenv-wordle.hf.space&quot;</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, **kwargs</span>) -&gt; <span class="hljs-literal">None</span> | <span class="hljs-built_in">str</span>:
        result = self.client.reset()
        <span class="hljs-comment"># The game returns cumulative feedback each turn (new text appended at the end), so</span>
        <span class="hljs-comment"># we store the previous full response and slice out only the newly appended part.</span>
        self._last_full_feedback = result.observation.messages[<span class="hljs-number">0</span>].content
        self.reward = <span class="hljs-number">0.0</span>
        self.done = <span class="hljs-literal">False</span>
        <span class="hljs-keyword">return</span> self._last_full_feedback

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">guess</span>(<span class="hljs-params">self, guess: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span>:
        <span class="hljs-string">&quot;&quot;&quot;
        Make a guess in the Wordle environment.

        Args:
            guess: The guessed word, formatted as &#x27;[abcde]&#x27;

        Returns:
            The feedback message from the environment.
        &quot;&quot;&quot;</span>
        <span class="hljs-keyword">if</span> self.done:
            <span class="hljs-keyword">raise</span> ValueError(<span class="hljs-string">&quot;Game over.&quot;</span>)
        result = self.client.step(TextArenaAction(message=guess))
        _full_feedback = result.observation.messages[<span class="hljs-number">0</span>].content
        <span class="hljs-comment"># Just take the new feedback since the last guess</span>
        feedback = _full_feedback[<span class="hljs-built_in">len</span>(self._last_full_feedback):]
        self._last_full_feedback = _full_feedback
        <span class="hljs-comment"># Penalize invalid moves</span>
        <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;You attempted an invalid move&quot;</span> <span class="hljs-keyword">in</span> feedback:
            self.reward = <span class="hljs-number">0.0</span>
        <span class="hljs-keyword">else</span>:
            self.reward = result.reward
        self.done = result.done
        <span class="hljs-keyword">return</span> feedback`,lang:"python",wrap:!1}}),L=new U({props:{title:"Define the reward function",local:"define-the-reward-function",headingTag:"h2"}}),P=new m({props:{code:"ZGVmJTIwcmV3YXJkX2Z1bmMoZW52aXJvbm1lbnRzJTJDJTIwKiprd2FyZ3MpJTIwLSUzRSUyMGxpc3QlNUJmbG9hdCU1RCUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QmVudi5yZXdhcmQlMjBmb3IlMjBlbnYlMjBpbiUyMGVudmlyb25tZW50cyU1RA==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">environments, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
    <span class="hljs-keyword">return</span> [env.reward <span class="hljs-keyword">for</span> env <span class="hljs-keyword">in</span> environments]`,lang:"python",wrap:!1}}),D=new U({props:{title:"Create the dataset",local:"create-the-dataset",headingTag:"h2"}}),ll=new m({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQSUwQWRhdGFzZXQlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglN0IlMjJwcm9tcHQlMjIlM0ElMjAlNUIlNUIlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjBwcm9tcHQlN0QlNUQlMjBmb3IlMjBfJTIwaW4lMjByYW5nZSgzMDAwKSU1RCU3RCk=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

dataset = Dataset.from_dict({<span class="hljs-string">&quot;prompt&quot;</span>: [[{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt}] <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3000</span>)]})`,lang:"python",wrap:!1}}),el=new U({props:{title:"Set GRPO Config",local:"set-grpo-config",headingTag:"h2"}}),nl=new m({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMEElMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyUXdlbiUyRlF3ZW4zLTEuN0IlMjIlMEFvdXRwdXRfZGlyJTIwJTNEJTIwJTIyd29yZGxlLWdycG8tUXdlbjMtMS43QiUyMiUwQSUwQWdycG9fY29uZmlnJTIwJTNEJTIwR1JQT0NvbmZpZyglMEElMjAlMjAlMjAlMjAlMjMlMjBUcmFpbmluZyUyMHNjaGVkdWxlJTIwJTJGJTIwb3B0aW1pemF0aW9uJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX2Vwb2NocyUzRDElMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMWUtNiUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDY0JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNEMSUyQyUwQSUyMCUyMCUyMCUyMHdhcm11cF9zdGVwcyUzRDEwJTJDJTBBJTIwJTIwJTIwJTIwb3B0aW0lM0QlMjJhZGFtd190b3JjaCUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9ncmFkX25vcm0lM0QxLjAlMkMlMEElMEElMjAlMjAlMjAlMjAlMjMlMjBHUlBPJTIwY29uZmlndXJhdGlvbiUwQSUyMCUyMCUyMCUyMG51bV9nZW5lcmF0aW9ucyUzRDIlMkMlMEElMjAlMjAlMjAlMjBtYXhfY29tcGxldGlvbl9sZW5ndGglM0QxMDI0JTJDJTBBJTIwJTIwJTIwJTIwbG9nX2NvbXBsZXRpb25zJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMG51bV9jb21wbGV0aW9uc190b19wcmludCUzRDIlMkMlMEElMjAlMjAlMjAlMjBjaGF0X3RlbXBsYXRlX2t3YXJncyUzRCU3QiUyMmVuYWJsZV90aGlua2luZyUyMiUzQSUyMEZhbHNlJTdEJTJDJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwdkxMTSUyMGNvbmZpZ3VyYXRpb24lMEElMjAlMjAlMjAlMjB1c2VfdmxsbSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjB2bGxtX21vZGUlM0QlMjJjb2xvY2F0ZSUyMiUyQyUwQSUyMCUyMCUyMCUyMHZsbG1fZ3B1X21lbW9yeV91dGlsaXphdGlvbiUzRDAuMTUlMkMlMEElMjAlMjAlMjAlMjB2bGxtX21heF9tb2RlbF9sZW5ndGglM0QzMDcyJTJDJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwTG9nZ2luZyUyMCUyRiUyMHJlcG9ydGluZyUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0RvdXRwdXRfZGlyJTJDJTBBJTIwJTIwJTIwJTIwcmVwb3J0X3RvJTNEJTIydHJhY2tpbyUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYWNraW9fc3BhY2VfaWQlM0RvdXRwdXRfZGlyJTJDJTBBJTIwJTIwJTIwJTIwbG9nZ2luZ19zdGVwcyUzRDElMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0ZXBzJTNEMTAlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3RvdGFsX2xpbWl0JTNEMSUyQyUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyME1lbW9yeSUyMG9wdGltaXphdGlvbiUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2NoZWNrcG9pbnRpbmclM0RUcnVlJTJDJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwSHViJTIwaW50ZWdyYXRpb24lMEElMjAlMjAlMjAlMjBwdXNoX3RvX2h1YiUzRFRydWUlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig

model_name = <span class="hljs-string">&quot;Qwen/Qwen3-1.7B&quot;</span>
output_dir = <span class="hljs-string">&quot;wordle-grpo-Qwen3-1.7B&quot;</span>

grpo_config = GRPOConfig(
    <span class="hljs-comment"># Training schedule / optimization</span>
    num_train_epochs=<span class="hljs-number">1</span>,
    learning_rate=<span class="hljs-number">1e-6</span>,
    gradient_accumulation_steps=<span class="hljs-number">64</span>,
    per_device_train_batch_size=<span class="hljs-number">1</span>,
    warmup_steps=<span class="hljs-number">10</span>,
    optim=<span class="hljs-string">&quot;adamw_torch&quot;</span>,
    max_grad_norm=<span class="hljs-number">1.0</span>,

    <span class="hljs-comment"># GRPO configuration</span>
    num_generations=<span class="hljs-number">2</span>,
    max_completion_length=<span class="hljs-number">1024</span>,
    log_completions=<span class="hljs-literal">True</span>,
    num_completions_to_print=<span class="hljs-number">2</span>,
    chat_template_kwargs={<span class="hljs-string">&quot;enable_thinking&quot;</span>: <span class="hljs-literal">False</span>},

    <span class="hljs-comment"># vLLM configuration</span>
    use_vllm=<span class="hljs-literal">True</span>,
    vllm_mode=<span class="hljs-string">&quot;colocate&quot;</span>,
    vllm_gpu_memory_utilization=<span class="hljs-number">0.15</span>,
    vllm_max_model_length=<span class="hljs-number">3072</span>,

    <span class="hljs-comment"># Logging / reporting</span>
    output_dir=output_dir,
    report_to=<span class="hljs-string">&quot;trackio&quot;</span>,
    trackio_space_id=output_dir,
    logging_steps=<span class="hljs-number">1</span>,
    save_steps=<span class="hljs-number">10</span>,
    save_total_limit=<span class="hljs-number">1</span>,

    <span class="hljs-comment"># Memory optimization</span>
    gradient_checkpointing=<span class="hljs-literal">True</span>,

    <span class="hljs-comment"># Hub integration</span>
    push_to_hub=<span class="hljs-literal">True</span>,
)`,lang:"python",wrap:!1}}),al=new U({props:{title:"Create the GRPOTrainer and start training",local:"create-the-grpotrainer-and-start-training",headingTag:"h2"}}),rl=new m({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTBBJTBBdHJhaW5lciUyMCUzRCUyMEdSUE9UcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWxfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRHJld2FyZF9mdW5jJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEZ3Jwb19jb25maWclMkMlMEElMjAlMjAlMjAlMjBlbnZpcm9ubWVudF9mYWN0b3J5JTNEV29yZGxlRW52JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer

trainer = GRPOTrainer(
    model=model_name,
    reward_funcs=reward_func,
    train_dataset=dataset,
    args=grpo_config,
    environment_factory=WordleEnv,
)`,lang:"python",wrap:!1}}),Jl=new m({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFncHVfc3RhdHMlMjAlM0QlMjB0b3JjaC5jdWRhLmdldF9kZXZpY2VfcHJvcGVydGllcygwKSUwQXN0YXJ0X2dwdV9tZW1vcnklMjAlM0QlMjByb3VuZCh0b3JjaC5jdWRhLm1heF9tZW1vcnlfcmVzZXJ2ZWQoKSUyMCUyRiUyMDEwMjQlMjAlMkYlMjAxMDI0JTIwJTJGJTIwMTAyNCUyQyUyMDMpJTBBbWF4X21lbW9yeSUyMCUzRCUyMHJvdW5kKGdwdV9zdGF0cy50b3RhbF9tZW1vcnklMjAlMkYlMjAxMDI0JTIwJTJGJTIwMTAyNCUyMCUyRiUyMDEwMjQlMkMlMjAzKSUwQSUwQXByaW50KGYlMjJHUFUlMjAlM0QlMjAlN0JncHVfc3RhdHMubmFtZSU3RC4lMjBNYXglMjBtZW1vcnklMjAlM0QlMjAlN0JtYXhfbWVtb3J5JTdEJTIwR0IuJTIyKSUwQXByaW50KGYlMjIlN0JzdGFydF9ncHVfbWVtb3J5JTdEJTIwR0IlMjBvZiUyMG1lbW9yeSUyMHJlc2VydmVkLiUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch

gpu_stats = torch.cuda.get_device_properties(<span class="hljs-number">0</span>)
start_gpu_memory = <span class="hljs-built_in">round</span>(torch.cuda.max_memory_reserved() / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span>, <span class="hljs-number">3</span>)
max_memory = <span class="hljs-built_in">round</span>(gpu_stats.total_memory / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span>, <span class="hljs-number">3</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;GPU = <span class="hljs-subst">{gpu_stats.name}</span>. Max memory = <span class="hljs-subst">{max_memory}</span> GB.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{start_gpu_memory}</span> GB of memory reserved.&quot;</span>)`,lang:"python",wrap:!1}}),Tl=new m({props:{code:"dHJhaW5lcl9zdGF0cyUyMCUzRCUyMHRyYWluZXIudHJhaW4oKQ==",highlighted:"trainer_stats = trainer.train()",lang:"python",wrap:!1}}),Ul=new m({props:{code:"dXNlZF9tZW1vcnklMjAlM0QlMjByb3VuZCh0b3JjaC5jdWRhLm1heF9tZW1vcnlfcmVzZXJ2ZWQoKSUyMCUyRiUyMDEwMjQlMjAlMkYlMjAxMDI0JTIwJTJGJTIwMTAyNCUyQyUyMDMpJTBBdXNlZF9tZW1vcnlfZm9yX3RyYWluaW5nJTIwJTNEJTIwcm91bmQodXNlZF9tZW1vcnklMjAtJTIwc3RhcnRfZ3B1X21lbW9yeSUyQyUyMDMpJTBBdXNlZF9wZXJjZW50YWdlJTIwJTNEJTIwcm91bmQodXNlZF9tZW1vcnklMjAlMkYlMjBtYXhfbWVtb3J5JTIwKiUyMDEwMCUyQyUyMDMpJTBBdHJhaW5pbmdfbWVtb3J5X3BlcmNlbnRhZ2UlMjAlM0QlMjByb3VuZCh1c2VkX21lbW9yeV9mb3JfdHJhaW5pbmclMjAlMkYlMjBtYXhfbWVtb3J5JTIwKiUyMDEwMCUyQyUyMDMpJTBBJTBBcHJpbnQoZiUyMiU3QnRyYWluZXJfc3RhdHMubWV0cmljcyU1Qid0cmFpbl9ydW50aW1lJyU1RCU3RCUyMHNlY29uZHMlMjB1c2VkJTIwZm9yJTIwdHJhaW5pbmcuJTIyKSUwQXByaW50KGYlMjIlN0Jyb3VuZCh0cmFpbmVyX3N0YXRzLm1ldHJpY3MlNUIndHJhaW5fcnVudGltZSclNUQlMkY2MCUyQyUyMDIpJTdEJTIwbWludXRlcyUyMHVzZWQlMjBmb3IlMjB0cmFpbmluZy4lMjIpJTBBcHJpbnQoZiUyMlBlYWslMjByZXNlcnZlZCUyMG1lbW9yeSUyMCUzRCUyMCU3QnVzZWRfbWVtb3J5JTdEJTIwR0IuJTIyKSUwQXByaW50KGYlMjJQZWFrJTIwcmVzZXJ2ZWQlMjBtZW1vcnklMjBmb3IlMjB0cmFpbmluZyUyMCUzRCUyMCU3QnVzZWRfbWVtb3J5X2Zvcl90cmFpbmluZyU3RCUyMEdCLiUyMiklMEFwcmludChmJTIyUGVhayUyMHJlc2VydmVkJTIwbWVtb3J5JTIwJTI1JTIwb2YlMjBtYXglMjBtZW1vcnklMjAlM0QlMjAlN0J1c2VkX3BlcmNlbnRhZ2UlN0QlMjAlMjUuJTIyKSUwQXByaW50KGYlMjJQZWFrJTIwcmVzZXJ2ZWQlMjBtZW1vcnklMjBmb3IlMjB0cmFpbmluZyUyMCUyNSUyMG9mJTIwbWF4JTIwbWVtb3J5JTIwJTNEJTIwJTdCdHJhaW5pbmdfbWVtb3J5X3BlcmNlbnRhZ2UlN0QlMjAlMjUuJTIyKQ==",highlighted:`used_memory = <span class="hljs-built_in">round</span>(torch.cuda.max_memory_reserved() / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span> / <span class="hljs-number">1024</span>, <span class="hljs-number">3</span>)
used_memory_for_training = <span class="hljs-built_in">round</span>(used_memory - start_gpu_memory, <span class="hljs-number">3</span>)
used_percentage = <span class="hljs-built_in">round</span>(used_memory / max_memory * <span class="hljs-number">100</span>, <span class="hljs-number">3</span>)
training_memory_percentage = <span class="hljs-built_in">round</span>(used_memory_for_training / max_memory * <span class="hljs-number">100</span>, <span class="hljs-number">3</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{trainer_stats.metrics[<span class="hljs-string">&#x27;train_runtime&#x27;</span>]}</span> seconds used for training.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{<span class="hljs-built_in">round</span>(trainer_stats.metrics[<span class="hljs-string">&#x27;train_runtime&#x27;</span>]/<span class="hljs-number">60</span>, <span class="hljs-number">2</span>)}</span> minutes used for training.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Peak reserved memory = <span class="hljs-subst">{used_memory}</span> GB.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Peak reserved memory for training = <span class="hljs-subst">{used_memory_for_training}</span> GB.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Peak reserved memory % of max memory = <span class="hljs-subst">{used_percentage}</span> %.&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Peak reserved memory for training % of max memory = <span class="hljs-subst">{training_memory_percentage}</span> %.&quot;</span>)`,lang:"python",wrap:!1}}),cl=new U({props:{title:"Save and push to Hub",local:"save-and-push-to-hub",headingTag:"h2"}}),jl=new m({props:{code:"dHJhaW5lci5zYXZlX21vZGVsKG91dHB1dF9kaXIpJTBBdHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:`trainer.save_model(output_dir)
trainer.push_to_hub()`,lang:"python",wrap:!1}}),dl=new U({props:{title:"Load the fine-tuned model and run inference",local:"load-the-fine-tuned-model-and-run-inference",headingTag:"h2"}}),hl=new m({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJzZXJnaW9wYW5pZWdvJTJGd29yZGxlLWdycG8tUXdlbjMtMS43QiUyMiUyMCUyMCUyMyUyMFJlcGxhY2UlMjB3aXRoJTIweW91ciUyMEhGJTIwdXNlcm5hbWUlMjBvciUyMG9yZ2FuaXphdGlvbiUwQSUwQWZpbmVfdHVuZWRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSUyQyUyMHRvcmNoX2R0eXBlJTNEJTIyZmxvYXQzMiUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;sergiopaniego/wordle-grpo-Qwen3-1.7B&quot;</span>  <span class="hljs-comment"># Replace with your HF username or organization</span>

fine_tuned_model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=<span class="hljs-string">&quot;float32&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
tokenizer = AutoTokenizer.from_pretrained(model_name)`,lang:"python",wrap:!1}}),Il=new m({props:{code:"aW1wb3J0JTIwanNvbiUwQSUwQSUwQWRlZiUyMHBsYXlfd29yZGxlKG1vZGVsJTJDJTIwdG9rZW5pemVyKSUzQSUwQSUyMCUyMCUyMCUyMGVudiUyMCUzRCUyMFdvcmRsZUVudigpJTBBJTIwJTIwJTIwJTIwaW5pdGlhbF9vYnNlcnZhdGlvbiUyMCUzRCUyMGVudi5yZXNldCgpJTBBJTBBJTIwJTIwJTIwJTIwcHJpbnQoJTIySW5pdGlhbCUyMG9ic2VydmF0aW9uJTNBJTIyKSUwQSUyMCUyMCUyMCUyMHByaW50KGluaXRpYWxfb2JzZXJ2YXRpb24pJTBBJTIwJTIwJTIwJTIwcHJpbnQoKSUwQSUwQSUyMCUyMCUyMCUyMG1lc3NhZ2VzJTIwJTNEJTIwJTVCJTdCJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwcHJvbXB0JTdEJTVEJTBBJTIwJTIwJTIwJTIwaWYlMjBpbml0aWFsX29ic2VydmF0aW9uJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWVzc2FnZXMuYXBwZW5kKCU3QiUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMGluaXRpYWxfb2JzZXJ2YXRpb24lN0QpJTBBJTBBJTIwJTIwJTIwJTIwZm9yJTIwdHVybiUyMGluJTIwcmFuZ2UoNiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGVudi5kb25lJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYnJlYWslMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcm9tcHRfdGV4dCUyMCUzRCUyMHRva2VuaXplci5hcHBseV9jaGF0X3RlbXBsYXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1lc3NhZ2VzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWRkX2dlbmVyYXRpb25fcHJvbXB0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRva2VuaXplJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmFibGVfdGhpbmtpbmclM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTVCcHJvbXB0X3RleHQlNUQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqbW9kZWxfaW5wdXRzJTJDJTIwbWF4X25ld190b2tlbnMlM0Q1MTIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0X2lkcyUyMCUzRCUyMGdlbmVyYXRlZF9pZHMlNUIwJTVEJTVCbGVuKG1vZGVsX2lucHV0cy5pbnB1dF9pZHMlNUIwJTVEKSUzQSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGdlbmVyYXRlZF90ZXh0JTIwJTNEJTIwdG9rZW5pemVyLmRlY29kZShvdXRwdXRfaWRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZiUyMlR1cm4lMjAlN0J0dXJuJTIwJTJCJTIwMSU3RCUyMC0lMjBNb2RlbCUyMG91dHB1dCUzQSUyMCU3QmdlbmVyYXRlZF90ZXh0JTdEJTIyKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFRyeSUyMHRvJTIwcGFyc2UlMjB0b29sJTIwY2FsbCUyMGZyb20lMjB0aGUlMjBnZW5lcmF0ZWQlMjB0ZXh0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJ5JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwVHJ5JTIwdG8lMjBleHRyYWN0JTIwYSUyMGd1ZXNzJTIwZnJvbSUyMHRvb2wlMjBjYWxsJTIwZm9ybWF0JTIwb3IlMjBicmFja2V0JTIwZm9ybWF0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjAlMjJndWVzcyUyMiUyMGluJTIwZ2VuZXJhdGVkX3RleHQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBQYXJzZSUyMEpTT04lMjB0b29sJTIwY2FsbCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXJ0JTIwJTNEJTIwZ2VuZXJhdGVkX3RleHQuaW5kZXgoJTIyJTdCJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVuZCUyMCUzRCUyMGdlbmVyYXRlZF90ZXh0LnJpbmRleCglMjIlN0QlMjIpJTIwJTJCJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFyZ3MlMjAlM0QlMjBqc29uLmxvYWRzKGdlbmVyYXRlZF90ZXh0JTVCc3RhcnQlM0FlbmQlNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjAlMjJhcmd1bWVudHMlMjIlMjBpbiUyMGFyZ3MlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhcmdzJTIwJTNEJTIwYXJncyU1QiUyMmFyZ3VtZW50cyUyMiU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGd1ZXNzX3dvcmQlMjAlM0QlMjBhcmdzLmdldCglMjJndWVzcyUyMiUyQyUyMCUyMiUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbHNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwRmFsbGJhY2slM0ElMjBleHRyYWN0JTIwZnJvbSUyMGJyYWNrZXRzJTIwJTVCd29yZCU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGltcG9ydCUyMHJlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWF0Y2glMjAlM0QlMjByZS5zZWFyY2gociUyMiU1QyU1QiglNUJhLXpBLVolNUQlN0I1JTdEKSU1QyU1RCUyMiUyQyUyMGdlbmVyYXRlZF90ZXh0KSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGd1ZXNzX3dvcmQlMjAlM0QlMjBtYXRjaC5ncm91cCgxKSUyMGlmJTIwbWF0Y2glMjBlbHNlJTIwZ2VuZXJhdGVkX3RleHQuc3RyaXAoKSU1QiUzQTUlNUQlMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWVkYmFjayUyMCUzRCUyMGVudi5ndWVzcyhmJTIyJTVCJTdCZ3Vlc3Nfd29yZCU3RCU1RCUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludChmJTIyJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR3Vlc3MlM0ElMjAlN0JndWVzc193b3JkJTdEJTIwJTdDJTIwUmV3YXJkJTNBJTIwJTdCZW52LnJld2FyZCU3RCUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludChmJTIyJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwRmVlZGJhY2slM0ElMjAlN0JmZWVkYmFjay5zdHJpcCgpJTdEJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtZXNzYWdlcy5hcHBlbmQoJTdCJTIycm9sZSUyMiUzQSUyMCUyMmFzc2lzdGFudCUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjBnZW5lcmF0ZWRfdGV4dCU3RCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtZXNzYWdlcy5hcHBlbmQoJTdCJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwZmVlZGJhY2slN0QpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXhjZXB0JTIwRXhjZXB0aW9uJTIwYXMlMjBlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEVycm9yJTNBJTIwJTdCZSU3RCUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBicmVhayUwQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJHYW1lJTIwZmluaXNoZWQhJTIwRmluYWwlMjByZXdhcmQlM0ElMjAlN0JlbnYucmV3YXJkJTdEJTIyKSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJEb25lJTNBJTIwJTdCZW52LmRvbmUlN0QlMjIp",highlighted:`<span class="hljs-keyword">import</span> json


<span class="hljs-keyword">def</span> <span class="hljs-title function_">play_wordle</span>(<span class="hljs-params">model, tokenizer</span>):
    env = WordleEnv()
    initial_observation = env.reset()

    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Initial observation:&quot;</span>)
    <span class="hljs-built_in">print</span>(initial_observation)
    <span class="hljs-built_in">print</span>()

    messages = [{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt}]
    <span class="hljs-keyword">if</span> initial_observation:
        messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: initial_observation})

    <span class="hljs-keyword">for</span> turn <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">6</span>):
        <span class="hljs-keyword">if</span> env.done:
            <span class="hljs-keyword">break</span>

        prompt_text = tokenizer.apply_chat_template(
            messages,
            add_generation_prompt=<span class="hljs-literal">True</span>,
            tokenize=<span class="hljs-literal">False</span>,
            enable_thinking=<span class="hljs-literal">False</span>,
        )
        model_inputs = tokenizer([prompt_text], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
        generated_ids = model.generate(**model_inputs, max_new_tokens=<span class="hljs-number">512</span>)
        output_ids = generated_ids[<span class="hljs-number">0</span>][<span class="hljs-built_in">len</span>(model_inputs.input_ids[<span class="hljs-number">0</span>]):]
        generated_text = tokenizer.decode(output_ids, skip_special_tokens=<span class="hljs-literal">True</span>)

        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Turn <span class="hljs-subst">{turn + <span class="hljs-number">1</span>}</span> - Model output: <span class="hljs-subst">{generated_text}</span>&quot;</span>)

        <span class="hljs-comment"># Try to parse tool call from the generated text</span>
        <span class="hljs-keyword">try</span>:
            <span class="hljs-comment"># Try to extract a guess from tool call format or bracket format</span>
            <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;guess&quot;</span> <span class="hljs-keyword">in</span> generated_text:
                <span class="hljs-comment"># Parse JSON tool call</span>
                start = generated_text.index(<span class="hljs-string">&quot;{&quot;</span>)
                end = generated_text.rindex(<span class="hljs-string">&quot;}&quot;</span>) + <span class="hljs-number">1</span>
                args = json.loads(generated_text[start:end])
                <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;arguments&quot;</span> <span class="hljs-keyword">in</span> args:
                    args = args[<span class="hljs-string">&quot;arguments&quot;</span>]
                guess_word = args.get(<span class="hljs-string">&quot;guess&quot;</span>, <span class="hljs-string">&quot;&quot;</span>)
            <span class="hljs-keyword">else</span>:
                <span class="hljs-comment"># Fallback: extract from brackets [word]</span>
                <span class="hljs-keyword">import</span> re
                <span class="hljs-keyword">match</span> = re.search(<span class="hljs-string">r&quot;\\[([a-zA-Z]{5})\\]&quot;</span>, generated_text)
                guess_word = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">1</span>) <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">else</span> generated_text.strip()[:<span class="hljs-number">5</span>]

            feedback = env.guess(<span class="hljs-string">f&quot;[<span class="hljs-subst">{guess_word}</span>]&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;         Guess: <span class="hljs-subst">{guess_word}</span> | Reward: <span class="hljs-subst">{env.reward}</span>&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;         Feedback: <span class="hljs-subst">{feedback.strip()}</span>&quot;</span>)
            <span class="hljs-built_in">print</span>()

            messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: generated_text})
            messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: feedback})
        <span class="hljs-keyword">except</span> Exception <span class="hljs-keyword">as</span> e:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;         Error: <span class="hljs-subst">{e}</span>&quot;</span>)
            <span class="hljs-keyword">break</span>

    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Game finished! Final reward: <span class="hljs-subst">{env.reward}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Done: <span class="hljs-subst">{env.done}</span>&quot;</span>)`,lang:"python",wrap:!1}}),Cl=new m({props:{code:"cGxheV93b3JkbGUoZmluZV90dW5lZF9tb2RlbCUyQyUyMHRva2VuaXplcik=",highlighted:"play_wordle(fine_tuned_model, tokenizer)",lang:"python",wrap:!1}}),fl=new Ws({props:{source:"https://github.com/huggingface/openenv/blob/main/docs/source/tutorials/wordle-grpo.md"}}),{c(){c=M("meta"),Bl=n(),gl=M("p"),Al=n(),y(j.$$.fragment),Wl=n(),y(d.$$.fragment),vl=n(),u=M("p"),u.innerHTML=Ee,Gl=n(),h=M("p"),h.innerHTML=He,_l=n(),I=M("p"),I.innerHTML=Ye,Rl=n(),b=M("ul"),b.innerHTML=Ne,kl=n(),C=M("p"),C.innerHTML=ze,Xl=n(),f=M("p"),f.innerHTML=xe,Ql=n(),g=M("ul"),g.innerHTML=Se,Vl=n(),Z=M("p"),Z.innerHTML=Fe,$l=n(),B=M("p"),B.innerHTML=Le,El=n(),A=M("ol"),A.innerHTML=qe,Hl=n(),W=M("p"),W.textContent=Ke,Yl=n(),y(v.$$.fragment),Nl=n(),G=M("p"),G.innerHTML=Pe,zl=n(),y(_.$$.fragment),xl=n(),y(R.$$.fragment),Sl=n(),k=M("p"),k.innerHTML=De,Fl=n(),y(X.$$.fragment),Ll=n(),y(Q.$$.fragment),ql=n(),V=M("p"),V.innerHTML=Oe,Kl=n(),y($.$$.fragment),Pl=n(),y(E.$$.fragment),Dl=n(),H=M("p"),H.innerHTML=ls,Ol=n(),Y=M("p"),Y.innerHTML=es,le=n(),N=M("ol"),N.innerHTML=ss,ee=n(),z=M("p"),z.innerHTML=ts,se=n(),x=M("p"),x.innerHTML=ns,te=n(),S=M("p"),S.innerHTML=as,ne=n(),y(F.$$.fragment),ae=n(),y(L.$$.fragment),Me=n(),q=M("p"),q.innerHTML=Ms,ie=n(),K=M("p"),K.textContent=is,pe=n(),y(P.$$.fragment),ye=n(),y(D.$$.fragment),re=n(),O=M("p"),O.textContent=ps,oe=n(),y(ll.$$.fragment),Je=n(),y(el.$$.fragment),we=n(),sl=M("p"),sl.innerHTML=ys,Te=n(),tl=M("p"),tl.innerHTML=rs,me=n(),y(nl.$$.fragment),Ue=n(),y(al.$$.fragment),ce=n(),Ml=M("p"),Ml.innerHTML=os,je=n(),il=M("p"),il.textContent=Js,de=n(),pl=M("ul"),pl.innerHTML=ws,ue=n(),yl=M("p"),yl.innerHTML=Ts,he=n(),y(rl.$$.fragment),Ie=n(),ol=M("p"),ol.textContent=ms,be=n(),y(Jl.$$.fragment),Ce=n(),wl=M("p"),wl.textContent=Us,fe=n(),y(Tl.$$.fragment),ge=n(),ml=M("p"),ml.textContent=cs,Ze=n(),y(Ul.$$.fragment),Be=n(),y(cl.$$.fragment),Ae=n(),y(jl.$$.fragment),We=n(),y(dl.$$.fragment),ve=n(),ul=M("p"),ul.innerHTML=js,Ge=n(),y(hl.$$.fragment),_e=n(),y(Il.$$.fragment),Re=n(),bl=M("p"),bl.textContent=ds,ke=n(),y(Cl.$$.fragment),Xe=n(),y(fl.$$.fragment),Qe=n(),Zl=M("p"),this.h()},l(l){const e=Zs("svelte-u9bgzb",document.head);c=i(e,"META",{name:!0,content:!0}),e.forEach(s),Bl=a(l),gl=i(l,"P",{}),us(gl).forEach(s),Al=a(l),r(j.$$.fragment,l),Wl=a(l),r(d.$$.fragment,l),vl=a(l),u=i(l,"P",{"data-svelte-h":!0}),p(u)!=="svelte-1fc6auj"&&(u.innerHTML=Ee),Gl=a(l),h=i(l,"P",{"data-svelte-h":!0}),p(h)!=="svelte-cos6ht"&&(h.innerHTML=He),_l=a(l),I=i(l,"P",{"data-svelte-h":!0}),p(I)!=="svelte-10kmmwt"&&(I.innerHTML=Ye),Rl=a(l),b=i(l,"UL",{"data-svelte-h":!0}),p(b)!=="svelte-1au2mwt"&&(b.innerHTML=Ne),kl=a(l),C=i(l,"P",{"data-svelte-h":!0}),p(C)!=="svelte-z0zrou"&&(C.innerHTML=ze),Xl=a(l),f=i(l,"P",{"data-svelte-h":!0}),p(f)!=="svelte-ecsij6"&&(f.innerHTML=xe),Ql=a(l),g=i(l,"UL",{"data-svelte-h":!0}),p(g)!=="svelte-ld1v3k"&&(g.innerHTML=Se),Vl=a(l),Z=i(l,"P",{"data-svelte-h":!0}),p(Z)!=="svelte-1ywqk01"&&(Z.innerHTML=Fe),$l=a(l),B=i(l,"P",{"data-svelte-h":!0}),p(B)!=="svelte-13ba4lb"&&(B.innerHTML=Le),El=a(l),A=i(l,"OL",{"data-svelte-h":!0}),p(A)!=="svelte-10sg7gd"&&(A.innerHTML=qe),Hl=a(l),W=i(l,"P",{"data-svelte-h":!0}),p(W)!=="svelte-152af3i"&&(W.textContent=Ke),Yl=a(l),r(v.$$.fragment,l),Nl=a(l),G=i(l,"P",{"data-svelte-h":!0}),p(G)!=="svelte-mzecib"&&(G.innerHTML=Pe),zl=a(l),r(_.$$.fragment,l),xl=a(l),r(R.$$.fragment,l),Sl=a(l),k=i(l,"P",{"data-svelte-h":!0}),p(k)!=="svelte-lyetiz"&&(k.innerHTML=De),Fl=a(l),r(X.$$.fragment,l),Ll=a(l),r(Q.$$.fragment,l),ql=a(l),V=i(l,"P",{"data-svelte-h":!0}),p(V)!=="svelte-698zkb"&&(V.innerHTML=Oe),Kl=a(l),r($.$$.fragment,l),Pl=a(l),r(E.$$.fragment,l),Dl=a(l),H=i(l,"P",{"data-svelte-h":!0}),p(H)!=="svelte-tiz0kv"&&(H.innerHTML=ls),Ol=a(l),Y=i(l,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-129airi"&&(Y.innerHTML=es),le=a(l),N=i(l,"OL",{"data-svelte-h":!0}),p(N)!=="svelte-10q3kdb"&&(N.innerHTML=ss),ee=a(l),z=i(l,"P",{"data-svelte-h":!0}),p(z)!=="svelte-17xvc4o"&&(z.innerHTML=ts),se=a(l),x=i(l,"P",{"data-svelte-h":!0}),p(x)!=="svelte-153e7gn"&&(x.innerHTML=ns),te=a(l),S=i(l,"P",{"data-svelte-h":!0}),p(S)!=="svelte-1k082v1"&&(S.innerHTML=as),ne=a(l),r(F.$$.fragment,l),ae=a(l),r(L.$$.fragment,l),Me=a(l),q=i(l,"P",{"data-svelte-h":!0}),p(q)!=="svelte-bofu78"&&(q.innerHTML=Ms),ie=a(l),K=i(l,"P",{"data-svelte-h":!0}),p(K)!=="svelte-s5bfmy"&&(K.textContent=is),pe=a(l),r(P.$$.fragment,l),ye=a(l),r(D.$$.fragment,l),re=a(l),O=i(l,"P",{"data-svelte-h":!0}),p(O)!=="svelte-x18exd"&&(O.textContent=ps),oe=a(l),r(ll.$$.fragment,l),Je=a(l),r(el.$$.fragment,l),we=a(l),sl=i(l,"P",{"data-svelte-h":!0}),p(sl)!=="svelte-1dfiz93"&&(sl.innerHTML=ys),Te=a(l),tl=i(l,"P",{"data-svelte-h":!0}),p(tl)!=="svelte-1hsds5z"&&(tl.innerHTML=rs),me=a(l),r(nl.$$.fragment,l),Ue=a(l),r(al.$$.fragment,l),ce=a(l),Ml=i(l,"P",{"data-svelte-h":!0}),p(Ml)!=="svelte-cp0d1t"&&(Ml.innerHTML=os),je=a(l),il=i(l,"P",{"data-svelte-h":!0}),p(il)!=="svelte-v455eo"&&(il.textContent=Js),de=a(l),pl=i(l,"UL",{"data-svelte-h":!0}),p(pl)!=="svelte-1m4xrmo"&&(pl.innerHTML=ws),ue=a(l),yl=i(l,"P",{"data-svelte-h":!0}),p(yl)!=="svelte-ooa6j6"&&(yl.innerHTML=Ts),he=a(l),r(rl.$$.fragment,l),Ie=a(l),ol=i(l,"P",{"data-svelte-h":!0}),p(ol)!=="svelte-1pw21xs"&&(ol.textContent=ms),be=a(l),r(Jl.$$.fragment,l),Ce=a(l),wl=i(l,"P",{"data-svelte-h":!0}),p(wl)!=="svelte-1b9poti"&&(wl.textContent=Us),fe=a(l),r(Tl.$$.fragment,l),ge=a(l),ml=i(l,"P",{"data-svelte-h":!0}),p(ml)!=="svelte-1t6bnxf"&&(ml.textContent=cs),Ze=a(l),r(Ul.$$.fragment,l),Be=a(l),r(cl.$$.fragment,l),Ae=a(l),r(jl.$$.fragment,l),We=a(l),r(dl.$$.fragment,l),ve=a(l),ul=i(l,"P",{"data-svelte-h":!0}),p(ul)!=="svelte-5ktyje"&&(ul.innerHTML=js),Ge=a(l),r(hl.$$.fragment,l),_e=a(l),r(Il.$$.fragment,l),Re=a(l),bl=i(l,"P",{"data-svelte-h":!0}),p(bl)!=="svelte-1lrqr39"&&(bl.textContent=ds),ke=a(l),r(Cl.$$.fragment,l),Xe=a(l),r(fl.$$.fragment,l),Qe=a(l),Zl=i(l,"P",{}),us(Zl).forEach(s),this.h()},h(){hs(c,"name","hf:doc:metadata"),hs(c,"content",Gs)},m(l,e){Bs(document.head,c),t(l,Bl,e),t(l,gl,e),t(l,Al,e),o(j,l,e),t(l,Wl,e),o(d,l,e),t(l,vl,e),t(l,u,e),t(l,Gl,e),t(l,h,e),t(l,_l,e),t(l,I,e),t(l,Rl,e),t(l,b,e),t(l,kl,e),t(l,C,e),t(l,Xl,e),t(l,f,e),t(l,Ql,e),t(l,g,e),t(l,Vl,e),t(l,Z,e),t(l,$l,e),t(l,B,e),t(l,El,e),t(l,A,e),t(l,Hl,e),t(l,W,e),t(l,Yl,e),o(v,l,e),t(l,Nl,e),t(l,G,e),t(l,zl,e),o(_,l,e),t(l,xl,e),o(R,l,e),t(l,Sl,e),t(l,k,e),t(l,Fl,e),o(X,l,e),t(l,Ll,e),o(Q,l,e),t(l,ql,e),t(l,V,e),t(l,Kl,e),o($,l,e),t(l,Pl,e),o(E,l,e),t(l,Dl,e),t(l,H,e),t(l,Ol,e),t(l,Y,e),t(l,le,e),t(l,N,e),t(l,ee,e),t(l,z,e),t(l,se,e),t(l,x,e),t(l,te,e),t(l,S,e),t(l,ne,e),o(F,l,e),t(l,ae,e),o(L,l,e),t(l,Me,e),t(l,q,e),t(l,ie,e),t(l,K,e),t(l,pe,e),o(P,l,e),t(l,ye,e),o(D,l,e),t(l,re,e),t(l,O,e),t(l,oe,e),o(ll,l,e),t(l,Je,e),o(el,l,e),t(l,we,e),t(l,sl,e),t(l,Te,e),t(l,tl,e),t(l,me,e),o(nl,l,e),t(l,Ue,e),o(al,l,e),t(l,ce,e),t(l,Ml,e),t(l,je,e),t(l,il,e),t(l,de,e),t(l,pl,e),t(l,ue,e),t(l,yl,e),t(l,he,e),o(rl,l,e),t(l,Ie,e),t(l,ol,e),t(l,be,e),o(Jl,l,e),t(l,Ce,e),t(l,wl,e),t(l,fe,e),o(Tl,l,e),t(l,ge,e),t(l,ml,e),t(l,Ze,e),o(Ul,l,e),t(l,Be,e),o(cl,l,e),t(l,Ae,e),o(jl,l,e),t(l,We,e),o(dl,l,e),t(l,ve,e),t(l,ul,e),t(l,Ge,e),o(hl,l,e),t(l,_e,e),o(Il,l,e),t(l,Re,e),t(l,bl,e),t(l,ke,e),o(Cl,l,e),t(l,Xe,e),o(fl,l,e),t(l,Qe,e),t(l,Zl,e),Ve=!0},p:bs,i(l){Ve||(J(j.$$.fragment,l),J(d.$$.fragment,l),J(v.$$.fragment,l),J(_.$$.fragment,l),J(R.$$.fragment,l),J(X.$$.fragment,l),J(Q.$$.fragment,l),J($.$$.fragment,l),J(E.$$.fragment,l),J(F.$$.fragment,l),J(L.$$.fragment,l),J(P.$$.fragment,l),J(D.$$.fragment,l),J(ll.$$.fragment,l),J(el.$$.fragment,l),J(nl.$$.fragment,l),J(al.$$.fragment,l),J(rl.$$.fragment,l),J(Jl.$$.fragment,l),J(Tl.$$.fragment,l),J(Ul.$$.fragment,l),J(cl.$$.fragment,l),J(jl.$$.fragment,l),J(dl.$$.fragment,l),J(hl.$$.fragment,l),J(Il.$$.fragment,l),J(Cl.$$.fragment,l),J(fl.$$.fragment,l),Ve=!0)},o(l){w(j.$$.fragment,l),w(d.$$.fragment,l),w(v.$$.fragment,l),w(_.$$.fragment,l),w(R.$$.fragment,l),w(X.$$.fragment,l),w(Q.$$.fragment,l),w($.$$.fragment,l),w(E.$$.fragment,l),w(F.$$.fragment,l),w(L.$$.fragment,l),w(P.$$.fragment,l),w(D.$$.fragment,l),w(ll.$$.fragment,l),w(el.$$.fragment,l),w(nl.$$.fragment,l),w(al.$$.fragment,l),w(rl.$$.fragment,l),w(Jl.$$.fragment,l),w(Tl.$$.fragment,l),w(Ul.$$.fragment,l),w(cl.$$.fragment,l),w(jl.$$.fragment,l),w(dl.$$.fragment,l),w(hl.$$.fragment,l),w(Il.$$.fragment,l),w(Cl.$$.fragment,l),w(fl.$$.fragment,l),Ve=!1},d(l){l&&(s(Bl),s(gl),s(Al),s(Wl),s(vl),s(u),s(Gl),s(h),s(_l),s(I),s(Rl),s(b),s(kl),s(C),s(Xl),s(f),s(Ql),s(g),s(Vl),s(Z),s($l),s(B),s(El),s(A),s(Hl),s(W),s(Yl),s(Nl),s(G),s(zl),s(xl),s(Sl),s(k),s(Fl),s(Ll),s(ql),s(V),s(Kl),s(Pl),s(Dl),s(H),s(Ol),s(Y),s(le),s(N),s(ee),s(z),s(se),s(x),s(te),s(S),s(ne),s(ae),s(Me),s(q),s(ie),s(K),s(pe),s(ye),s(re),s(O),s(oe),s(Je),s(we),s(sl),s(Te),s(tl),s(me),s(Ue),s(ce),s(Ml),s(je),s(il),s(de),s(pl),s(ue),s(yl),s(he),s(Ie),s(ol),s(be),s(Ce),s(wl),s(fe),s(ge),s(ml),s(Ze),s(Be),s(Ae),s(We),s(ve),s(ul),s(Ge),s(_e),s(Re),s(bl),s(ke),s(Xe),s(Qe),s(Zl)),s(c),T(j,l),T(d,l),T(v,l),T(_,l),T(R,l),T(X,l),T(Q,l),T($,l),T(E,l),T(F,l),T(L,l),T(P,l),T(D,l),T(ll,l),T(el,l),T(nl,l),T(al,l),T(rl,l),T(Jl,l),T(Tl,l),T(Ul,l),T(cl,l),T(jl,l),T(dl,l),T(hl,l),T(Il,l),T(Cl,l),T(fl,l)}}}const Gs='{"title":"OpenEnv Wordle with GRPO using TRL","local":"openenv-wordle-with-grpo-using-trl","sections":[{"title":"Install dependencies","local":"install-dependencies","sections":[{"title":"Log in to Hugging Face","local":"log-in-to-hugging-face","sections":[],"depth":3}],"depth":2},{"title":"Define the system prompt","local":"define-the-system-prompt","sections":[],"depth":2},{"title":"Define the environment","local":"define-the-environment","sections":[],"depth":2},{"title":"Define the reward function","local":"define-the-reward-function","sections":[],"depth":2},{"title":"Create the dataset","local":"create-the-dataset","sections":[],"depth":2},{"title":"Set GRPO Config","local":"set-grpo-config","sections":[],"depth":2},{"title":"Create the GRPOTrainer and start training","local":"create-the-grpotrainer-and-start-training","sections":[],"depth":2},{"title":"Save and push to Hub","local":"save-and-push-to-hub","sections":[],"depth":2},{"title":"Load the fine-tuned model and run inference","local":"load-the-fine-tuned-model-and-run-inference","sections":[],"depth":2}],"depth":1}';function _s($e){return Cs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Vs extends fs{constructor(c){super(),gs(this,c,_s,vs,Is,{})}}export{Vs as component};