openenv/pr_813/en/_app/immutable/nodes/61.2da2ea0b.js

import{s as ga,n as Ba,o as fa}from"../chunks/scheduler.2b22cead.js";import{S as Za,i as Ga,e as r,s as t,c as M,h as Aa,a as j,d as a,b as e,f as ba,g as p,j as o,k as Os,l as Ea,m as n,n as y,t as i,o as J,p as U}from"../chunks/index.1a0e8013.js";import{C as Qa,H as T,E as Wa}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f5f19e58.js";import{C as w}from"../chunks/CodeBlock.640a2a71.js";function Ra(la){let c,kl,Wl,$l,u,Sl,I,vl,d,sa=`This tutorial covers training a language model to play the 2048 game using
reinforcement learning with GRPO (Group Relative Policy Optimization).`,Vl,m,aa="<p><strong>Time</strong>: ~45 minutes | <strong>Difficulty</strong>: Advanced | <strong>GPU Required</strong>: Yes (T4 or better)</p>",_l,h,Xl,C,na="<li><strong>Model Setup</strong>: Load and configure LLMs with Unsloth for efficient RL</li> <li><strong>Environment Connection</strong>: Connect to the 2048 OpenEnv environment</li> <li><strong>Reward Design</strong>: Create effective reward functions</li> <li><strong>GRPO Training</strong>: Train models with reinforcement learning</li> <li><strong>Deployment</strong>: Save and deploy trained models</li>",Nl,b,Fl,g,ta=`Before starting this tutorial, you should have completed the
<a href="index">Getting Started</a> series to understand:`,Yl,B,ea="<li>How OpenEnv environments work</li> <li>The reset/step/state API pattern</li> <li>How to connect to environments</li>",zl,f,Ma="You’ll also need:",ql,Z,pa="<li>A GPU (free T4 on Google Colab works)</li> <li>Basic understanding of PyTorch</li> <li>~30 minutes for training</li>",Hl,G,xl,A,Ll,E,Dl,Q,Kl,W,Pl,R,Ol,k,ya="We use Unsloth for memory-efficient training with LoRA adapters.",ls,$,ss,S,as,v,ns,V,ts,_,es,X,Ms,N,ia="2048 is a sliding puzzle game where you combine tiles to reach 2048.",ps,F,Ja="<strong>Actions:</strong>",ys,Y,Ua="<li><code>0</code> = UP</li> <li><code>1</code> = RIGHT</li> <li><code>2</code> = DOWN</li> <li><code>3</code> = LEFT</li>",is,z,Ta="<strong>Goal:</strong> Create a tile with value 2048 (or higher!)",Js,q,Us,H,Ts,x,rs,L,js,D,os,K,ra="The reward function is crucial for RL. We consider:",ws,P,ja="<li><strong>Success</strong>: Did we reach 2048?</li> <li><strong>Progress</strong>: What’s the highest tile achieved?</li> <li><strong>Code Quality</strong>: Did the generated code execute correctly?</li>",cs,O,ms,ll,us,sl,Is,al,oa="We’ll train the model to generate Python strategy functions.",ds,nl,hs,tl,Cs,el,bs,Ml,gs,pl,Bs,yl,wa="GRPO (Group Relative Policy Optimization) is optimized for language models.",fs,il,Zs,Jl,Gs,Ul,As,Tl,Es,rl,Qs,jl,ca="After training, save and deploy your model.",Ws,ol,Rs,wl,ks,cl,$s,ml,Ss,ul,vs,Il,Vs,dl,_s,hl,ma="Be aware of potential reward hacking strategies:",Xs,Cl,ua="<li><strong>Code that modifies rewards</strong> - Run in sandboxed environment</li> <li><strong>Infinite loops</strong> - Set execution timeouts</li> <li><strong>Memory exhaustion</strong> - Limit resource usage</li>",Ns,bl,Fs,gl,Ys,Bl,Ia="In this tutorial, you learned:",zs,fl,da="<li><strong>Model Setup</strong>: Loading LLMs with Unsloth and LoRA</li> <li><strong>Environment Connection</strong>: Using OpenEnv’s 2048 environment</li> <li><strong>Reward Design</strong>: Creating balanced reward functions</li> <li><strong>GRPO Training</strong>: Training with reinforcement learning</li> <li><strong>Deployment</strong>: Saving and sharing trained models</li>",qs,Zl,Hs,Gl,ha="<li>Try different model architectures</li> <li>Experiment with reward function designs</li> <li>Train on other OpenEnv environments</li> <li>Share your trained models on Hugging Face Hub!</li>",xs,Al,Ls,El,Ca='<li><a href="index">OpenEnv Getting Started</a></li> <li><a href="../getting_started/environment-builder">Building Custom Environments</a></li> <li><a href="https://huggingface.co/docs/trl/grpo_trainer" rel="nofollow">GRPO Documentation</a></li> <li><a href="https://github.com/unslothai/unsloth" rel="nofollow">Unsloth Documentation</a></li>',Ds,Ql,Ks,Rl,Ps;return u=new Qa({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),I=new T({props:{title:"RL Training with OpenEnv: 2048 Game",local:"rl-training-with-openenv-2048-game",headingTag:"h1"}}),h=new T({props:{title:"What You’ll Learn",local:"what-youll-learn",headingTag:"h2"}}),b=new T({props:{title:"Prerequisites",local:"prerequisites",headingTag:"h2"}}),G=new T({props:{title:"Part 1: Environment Setup",local:"part-1-environment-setup",headingTag:"h2"}}),A=new T({props:{title:"Installation",local:"installation",headingTag:"h3"}}),E=new w({props:{code:"JTIzJTIwSW5zdGFsbCUyMHJlcXVpcmVkJTIwcGFja2FnZXMlMEEhcGlwJTIwaW5zdGFsbCUyMC1xJTIwdW5zbG90aCUyMG9wZW5lbnYlMjB0cmwlMEElMEElMjMlMjBGb3IlMjBHb29nbGUlMjBDb2xhYiUyQyUyMGFsc28lMjBydW4lM0ElMEEhcGlwJTIwaW5zdGFsbCUyMC1xJTIwJTIydW5zbG90aCU1QmNvbGFiLW5ldyU1RCUyMCU0MCUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRnVuc2xvdGhhaSUyRnVuc2xvdGguZ2l0JTIy",highlighted:`<span class="hljs-comment"># Install required packages</span>
!pip install -q unsloth openenv trl

<span class="hljs-comment"># For Google Colab, also run:</span>
!pip install -q <span class="hljs-string">&quot;unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git&quot;</span>`,lang:"bash",wrap:!1}}),Q=new T({props:{title:"Imports",local:"imports",headingTag:"h3"}}),W=new w({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGF0YWNsYXNzZXMlMjBpbXBvcnQlMjBkYXRhY2xhc3MlMEFmcm9tJTIwdHlwaW5nJTIwaW1wb3J0JTIwTGlzdCUyQyUyME9wdGlvbmFsJTJDJTIwRGljdCUyQyUyMEFueSUwQWltcG9ydCUyMHJhbmRvbSUwQSUwQSUyMyUyMENoZWNrJTIwR1BVJTIwYXZhaWxhYmlsaXR5JTBBcHJpbnQoZiUyMkdQVSUyMEF2YWlsYWJsZSUzQSUyMCU3QnRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklN0QlMjIpJTBBaWYlMjB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMkdQVSUzQSUyMCU3QnRvcmNoLmN1ZGEuZ2V0X2RldmljZV9uYW1lKDApJTdEJTIyKSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJNZW1vcnklM0ElMjAlN0J0b3JjaC5jdWRhLmdldF9kZXZpY2VfcHJvcGVydGllcygwKS50b3RhbF9tZW1vcnklMjAlMkYlMjAxZTklM0EuMWYlN0QlMjBHQiUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> dataclasses <span class="hljs-keyword">import</span> dataclass
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">List</span>, <span class="hljs-type">Optional</span>, <span class="hljs-type">Dict</span>, <span class="hljs-type">Any</span>
<span class="hljs-keyword">import</span> random

<span class="hljs-comment"># Check GPU availability</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;GPU Available: <span class="hljs-subst">{torch.cuda.is_available()}</span>&quot;</span>)
<span class="hljs-keyword">if</span> torch.cuda.is_available():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;GPU: <span class="hljs-subst">{torch.cuda.get_device_name(<span class="hljs-number">0</span>)}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Memory: <span class="hljs-subst">{torch.cuda.get_device_properties(<span class="hljs-number">0</span>).total_memory / <span class="hljs-number">1e9</span>:<span class="hljs-number">.1</span>f}</span> GB&quot;</span>)`,lang:"python",wrap:!1}}),R=new T({props:{title:"Part 2: Model Configuration",local:"part-2-model-configuration",headingTag:"h2"}}),$=new T({props:{title:"Configuration Classes",local:"configuration-classes",headingTag:"h3"}}),S=new w({props:{code:"JTQwZGF0YWNsYXNzJTBBY2xhc3MlMjBNb2RlbENvbmZpZyUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMkNvbmZpZ3VyYXRpb24lMjBmb3IlMjBsb2FkaW5nJTIwTExNJTIwbW9kZWxzLiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0ElMjBzdHIlMjAlM0QlMjAlMjJ1bnNsb3RoJTJGUXdlbjIuNS0xLjVCJTIyJTBBJTIwJTIwJTIwJTIwbWF4X3NlcV9sZW5ndGglM0ElMjBpbnQlMjAlM0QlMjA3NjglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0ElMjBib29sJTIwJTNEJTIwVHJ1ZSUwQSUyMCUyMCUyMCUyMGR0eXBlJTNBJTIwT3B0aW9uYWwlNUJzdHIlNUQlMjAlM0QlMjBOb25lJTIwJTIwJTIzJTIwQXV0by1kZXRlY3QlMEElMEElMEElNDBkYXRhY2xhc3MlMEFjbGFzcyUyMExvUkFDb25maWclM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjJDb25maWd1cmF0aW9uJTIwZm9yJTIwTG9SQSUyMGZpbmUtdHVuaW5nLiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMHIlM0ElMjBpbnQlMjAlM0QlMjAxNiUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0ElMjBpbnQlMjAlM0QlMjAzMiUwQSUyMCUyMCUyMCUyMHRhcmdldF9tb2R1bGVzJTNBJTIwTGlzdCU1QnN0ciU1RCUyMCUzRCUyME5vbmUlMEElMjAlMjAlMjAlMjBsb3JhX2Ryb3BvdXQlM0ElMjBmbG9hdCUyMCUzRCUyMDAuMCUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9fcG9zdF9pbml0X18oc2VsZiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHNlbGYudGFyZ2V0X21vZHVsZXMlMjBpcyUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLnRhcmdldF9tb2R1bGVzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycV9wcm9qJTIyJTJDJTIwJTIya19wcm9qJTIyJTJDJTIwJTIydl9wcm9qJTIyJTJDJTIwJTIyb19wcm9qJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZ2F0ZV9wcm9qJTIyJTJDJTIwJTIydXBfcHJvaiUyMiUyQyUyMCUyMmRvd25fcHJvaiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RA==",highlighted:`<span class="hljs-meta">@dataclass</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">ModelConfig</span>:
    <span class="hljs-string">&quot;&quot;&quot;Configuration for loading LLM models.&quot;&quot;&quot;</span>
    model_name: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;unsloth/Qwen2.5-1.5B&quot;</span>
    max_seq_length: <span class="hljs-built_in">int</span> = <span class="hljs-number">768</span>
    load_in_4bit: <span class="hljs-built_in">bool</span> = <span class="hljs-literal">True</span>
    dtype: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">str</span>] = <span class="hljs-literal">None</span>  <span class="hljs-comment"># Auto-detect</span>


<span class="hljs-meta">@dataclass</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">LoRAConfig</span>:
    <span class="hljs-string">&quot;&quot;&quot;Configuration for LoRA fine-tuning.&quot;&quot;&quot;</span>
    r: <span class="hljs-built_in">int</span> = <span class="hljs-number">16</span>
    lora_alpha: <span class="hljs-built_in">int</span> = <span class="hljs-number">32</span>
    target_modules: <span class="hljs-type">List</span>[<span class="hljs-built_in">str</span>] = <span class="hljs-literal">None</span>
    lora_dropout: <span class="hljs-built_in">float</span> = <span class="hljs-number">0.0</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__post_init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-keyword">if</span> self.target_modules <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
            self.target_modules = [
                <span class="hljs-string">&quot;q_proj&quot;</span>, <span class="hljs-string">&quot;k_proj&quot;</span>, <span class="hljs-string">&quot;v_proj&quot;</span>, <span class="hljs-string">&quot;o_proj&quot;</span>,
                <span class="hljs-string">&quot;gate_proj&quot;</span>, <span class="hljs-string">&quot;up_proj&quot;</span>, <span class="hljs-string">&quot;down_proj&quot;</span>,
            ]`,lang:"python",wrap:!1}}),v=new T({props:{title:"Loading the Model",local:"loading-the-model",headingTag:"h3"}}),V=new w({props:{code:"ZnJvbSUyMHVuc2xvdGglMjBpbXBvcnQlMjBGYXN0TGFuZ3VhZ2VNb2RlbCUwQSUwQSUyMyUyMENyZWF0ZSUyMGNvbmZpZ3VyYXRpb25zJTBBbW9kZWxfY29uZmlnJTIwJTNEJTIwTW9kZWxDb25maWcoKSUwQWxvcmFfY29uZmlnJTIwJTNEJTIwTG9SQUNvbmZpZygpJTBBJTBBJTIzJTIwTG9hZCUyMG1vZGVsJTBBbW9kZWwlMkMlMjB0b2tlbml6ZXIlMjAlM0QlMjBGYXN0TGFuZ3VhZ2VNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwbW9kZWxfbmFtZSUzRG1vZGVsX2NvbmZpZy5tb2RlbF9uYW1lJTJDJTBBJTIwJTIwJTIwJTIwbWF4X3NlcV9sZW5ndGglM0Rtb2RlbF9jb25maWcubWF4X3NlcV9sZW5ndGglMkMlMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0Rtb2RlbF9jb25maWcubG9hZF9pbl80Yml0JTJDJTBBJTIwJTIwJTIwJTIwZHR5cGUlM0Rtb2RlbF9jb25maWcuZHR5cGUlMkMlMEEpJTBBJTBBJTIzJTIwQXBwbHklMjBMb1JBJTIwYWRhcHRlcnMlMEFtb2RlbCUyMCUzRCUyMEZhc3RMYW5ndWFnZU1vZGVsLmdldF9wZWZ0X21vZGVsKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwciUzRGxvcmFfY29uZmlnLnIlMkMlMEElMjAlMjAlMjAlMjB0YXJnZXRfbW9kdWxlcyUzRGxvcmFfY29uZmlnLnRhcmdldF9tb2R1bGVzJTJDJTBBJTIwJTIwJTIwJTIwbG9yYV9hbHBoYSUzRGxvcmFfY29uZmlnLmxvcmFfYWxwaGElMkMlMEElMjAlMjAlMjAlMjBsb3JhX2Ryb3BvdXQlM0Rsb3JhX2NvbmZpZy5sb3JhX2Ryb3BvdXQlMkMlMEElMjAlMjAlMjAlMjBiaWFzJTNEJTIybm9uZSUyMiUyQyUwQSUyMCUyMCUyMCUyMHVzZV9ncmFkaWVudF9jaGVja3BvaW50aW5nJTNEJTIydW5zbG90aCUyMiUyQyUwQSUyMCUyMCUyMCUyMHJhbmRvbV9zdGF0ZSUzRDQyJTJDJTBBKSUwQSUwQSUyMyUyMENoZWNrJTIwcGFyYW1ldGVyJTIwY291bnRzJTBBdHJhaW5hYmxlJTIwJTNEJTIwc3VtKHAubnVtZWwoKSUyMGZvciUyMHAlMjBpbiUyMG1vZGVsLnBhcmFtZXRlcnMoKSUyMGlmJTIwcC5yZXF1aXJlc19ncmFkKSUwQXRvdGFsJTIwJTNEJTIwc3VtKHAubnVtZWwoKSUyMGZvciUyMHAlMjBpbiUyMG1vZGVsLnBhcmFtZXRlcnMoKSklMEFwcmludChmJTIyVHJhaW5hYmxlJTNBJTIwJTdCdHJhaW5hYmxlJTNBJTJDJTdEJTIwJTJGJTIwJTdCdG90YWwlM0ElMkMlN0QlMjAoJTdCdHJhaW5hYmxlJTJGdG90YWwqMTAwJTNBLjJmJTdEJTI1KSUyMik=",highlighted:`<span class="hljs-keyword">from</span> unsloth <span class="hljs-keyword">import</span> FastLanguageModel

<span class="hljs-comment"># Create configurations</span>
model_config = ModelConfig()
lora_config = LoRAConfig()

<span class="hljs-comment"># Load model</span>
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name=model_config.model_name,
    max_seq_length=model_config.max_seq_length,
    load_in_4bit=model_config.load_in_4bit,
    dtype=model_config.dtype,
)

<span class="hljs-comment"># Apply LoRA adapters</span>
model = FastLanguageModel.get_peft_model(
    model,
    r=lora_config.r,
    target_modules=lora_config.target_modules,
    lora_alpha=lora_config.lora_alpha,
    lora_dropout=lora_config.lora_dropout,
    bias=<span class="hljs-string">&quot;none&quot;</span>,
    use_gradient_checkpointing=<span class="hljs-string">&quot;unsloth&quot;</span>,
    random_state=<span class="hljs-number">42</span>,
)

<span class="hljs-comment"># Check parameter counts</span>
trainable = <span class="hljs-built_in">sum</span>(p.numel() <span class="hljs-keyword">for</span> p <span class="hljs-keyword">in</span> model.parameters() <span class="hljs-keyword">if</span> p.requires_grad)
total = <span class="hljs-built_in">sum</span>(p.numel() <span class="hljs-keyword">for</span> p <span class="hljs-keyword">in</span> model.parameters())
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Trainable: <span class="hljs-subst">{trainable:,}</span> / <span class="hljs-subst">{total:,}</span> (<span class="hljs-subst">{trainable/total*<span class="hljs-number">100</span>:<span class="hljs-number">.2</span>f}</span>%)&quot;</span>)`,lang:"python",wrap:!1}}),_=new T({props:{title:"Part 3: The 2048 Environment",local:"part-3-the-2048-environment",headingTag:"h2"}}),X=new T({props:{title:"Game Overview",local:"game-overview",headingTag:"h3"}}),q=new T({props:{title:"Connecting to the Environment",local:"connecting-to-the-environment",headingTag:"h3"}}),H=new w({props:{code:"ZnJvbSUyMGVudnMub3BlbnNwaWVsX2VudiUyMGltcG9ydCUyME9wZW5TcGllbEVudiUyQyUyME9wZW5TcGllbEFjdGlvbiUwQSUwQSUyMyUyMENvbm5lY3QlMjB0byUyMDIwNDglMjBlbnZpcm9ubWVudCUwQSUyMyUyME9wdGlvbiUyMDElM0ElMjBGcm9tJTIwSHViJTBBZW52JTIwJTNEJTIwT3BlblNwaWVsRW52LmZyb21faHViKCUyMm9wZW5lbnYlMkZvcGVuc3BpZWwtZW52JTIyKSUwQSUwQSUyMyUyME9wdGlvbiUyMDIlM0ElMjBGcm9tJTIwcnVubmluZyUyMHNlcnZlciUwQSUyMyUyMGVudiUyMCUzRCUyME9wZW5TcGllbEVudihiYXNlX3VybCUzRCUyMmh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDAwJTIyKSUwQSUwQSUyMyUyMFRlc3QlMjBjb25uZWN0aW9uJTBBd2l0aCUyMGVudiUzQSUwQSUyMCUyMCUyMCUyMHJlc3VsdCUyMCUzRCUyMGVudi5yZXNldCgpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMkdhbWUlMjBzdGFydGVkISUyMiklMEElMjAlMjAlMjAlMjBwcmludChmJTIyTGVnYWwlMjBhY3Rpb25zJTNBJTIwJTdCcmVzdWx0Lm9ic2VydmF0aW9uLmxlZ2FsX2FjdGlvbnMlN0QlMjIpJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwVGFrZSUyMGElMjB0ZXN0JTIwYWN0aW9uJTBBJTIwJTIwJTIwJTIwYWN0aW9uJTIwJTNEJTIwT3BlblNwaWVsQWN0aW9uKGFjdGlvbl9pZCUzRDAlMkMlMjBnYW1lX25hbWUlM0QlMjIyMDQ4JTIyKSUwQSUyMCUyMCUyMCUyMHJlc3VsdCUyMCUzRCUyMGVudi5zdGVwKGFjdGlvbiklMEElMjAlMjAlMjAlMjBwcmludChmJTIyQWZ0ZXIlMjBVUCUzQSUyMHJld2FyZCUzRCU3QnJlc3VsdC5yZXdhcmQlN0QlMkMlMjBkb25lJTNEJTdCcmVzdWx0LmRvbmUlN0QlMjIp",highlighted:`<span class="hljs-keyword">from</span> envs.openspiel_env <span class="hljs-keyword">import</span> OpenSpielEnv, OpenSpielAction

<span class="hljs-comment"># Connect to 2048 environment</span>
<span class="hljs-comment"># Option 1: From Hub</span>
env = OpenSpielEnv.from_hub(<span class="hljs-string">&quot;openenv/openspiel-env&quot;</span>)

<span class="hljs-comment"># Option 2: From running server</span>
<span class="hljs-comment"># env = OpenSpielEnv(base_url=&quot;http://localhost:8000&quot;)</span>

<span class="hljs-comment"># Test connection</span>
<span class="hljs-keyword">with</span> env:
    result = env.reset()
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Game started!&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Legal actions: <span class="hljs-subst">{result.observation.legal_actions}</span>&quot;</span>)

    <span class="hljs-comment"># Take a test action</span>
    action = OpenSpielAction(action_id=<span class="hljs-number">0</span>, game_name=<span class="hljs-string">&quot;2048&quot;</span>)
    result = env.step(action)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;After UP: reward=<span class="hljs-subst">{result.reward}</span>, done=<span class="hljs-subst">{result.done}</span>&quot;</span>)`,lang:"python",wrap:!1}}),x=new T({props:{title:"Board Utilities",local:"board-utilities",headingTag:"h3"}}),L=new w({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBZnJvbSUyMHR5cGluZyUyMGltcG9ydCUyMExpc3QlMEElMEFkZWYlMjBpbmZvX3N0YXRlX3RvX2JvYXJkKGluZm9fc3RhdGUlM0ElMjBMaXN0JTVCaW50JTVEJTJDJTIwc2l6ZSUzQSUyMGludCUyMCUzRCUyMDQpJTIwLSUzRSUyMExpc3QlNUJMaXN0JTVCaW50JTVEJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyQ29udmVydCUyMGZsYXQlMjBpbmZvX3N0YXRlJTIwdG8lMjAyRCUyMGJvYXJkLiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMHJldHVybiUyMG5wLmFycmF5KGluZm9fc3RhdGUlMkMlMjBkdHlwZSUzRGludCkucmVzaGFwZShzaXplJTJDJTIwc2l6ZSkudG9saXN0KCklMEElMEFkZWYlMjByZW5kZXJfYm9hcmQoYm9hcmQlM0ElMjBMaXN0JTVCTGlzdCU1QmludCU1RCU1RCklMjAtJTNFJTIwc3RyJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyUmVuZGVyJTIwYm9hcmQlMjBhcyUyMEFTQ0lJJTIwc3RyaW5nLiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMGxpbmVzJTIwJTNEJTIwJTVCJTIyJTJCLS0tLS0tJTIyJTIwKiUyMGxlbihib2FyZCU1QjAlNUQpJTIwJTJCJTIwJTIyJTJCJTIyJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwcm93JTIwaW4lMjBib2FyZCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNlbGxzJTIwJTNEJTIwJTVCZiUyMiU3QnYlM0E1ZCU3RCUyMiUyMGlmJTIwdiUyMCUzRSUyMDAlMjBlbHNlJTIwJTIyJTIwJTIwJTIwJTIwLiUyMiUyMGZvciUyMHYlMjBpbiUyMHJvdyU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxpbmVzLmFwcGVuZCglMjIlN0MlMjIlMjAlMkIlMjAlMjIlMjAlN0MlMjIuam9pbihjZWxscyklMjAlMkIlMjAlMjIlMjAlN0MlMjIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGluZXMuYXBwZW5kKCUyMiUyQi0tLS0tLSUyMiUyMColMjBsZW4ocm93KSUyMCUyQiUyMCUyMiUyQiUyMiklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlMjIlNUNuJTIyLmpvaW4obGluZXMpJTBBJTBBZGVmJTIwZ2V0X21heF90aWxlKGJvYXJkJTNBJTIwTGlzdCU1Qkxpc3QlNUJpbnQlNUQlNUQpJTIwLSUzRSUyMGludCUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMkdldCUyMGhpZ2hlc3QlMjB0aWxlJTIwdmFsdWUuJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwbWF4KGNlbGwlMjBmb3IlMjByb3clMjBpbiUyMGJvYXJkJTIwZm9yJTIwY2VsbCUyMGluJTIwcm93KQ==",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">List</span>

<span class="hljs-keyword">def</span> <span class="hljs-title function_">info_state_to_board</span>(<span class="hljs-params">info_state: <span class="hljs-type">List</span>[<span class="hljs-built_in">int</span>], size: <span class="hljs-built_in">int</span> = <span class="hljs-number">4</span></span>) -&gt; <span class="hljs-type">List</span>[<span class="hljs-type">List</span>[<span class="hljs-built_in">int</span>]]:
    <span class="hljs-string">&quot;&quot;&quot;Convert flat info_state to 2D board.&quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> np.array(info_state, dtype=<span class="hljs-built_in">int</span>).reshape(size, size).tolist()

<span class="hljs-keyword">def</span> <span class="hljs-title function_">render_board</span>(<span class="hljs-params">board: <span class="hljs-type">List</span>[<span class="hljs-type">List</span>[<span class="hljs-built_in">int</span>]]</span>) -&gt; <span class="hljs-built_in">str</span>:
    <span class="hljs-string">&quot;&quot;&quot;Render board as ASCII string.&quot;&quot;&quot;</span>
    lines = [<span class="hljs-string">&quot;+------&quot;</span> * <span class="hljs-built_in">len</span>(board[<span class="hljs-number">0</span>]) + <span class="hljs-string">&quot;+&quot;</span>]
    <span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> board:
        cells = [<span class="hljs-string">f&quot;<span class="hljs-subst">{v:5d}</span>&quot;</span> <span class="hljs-keyword">if</span> v &gt; <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;    .&quot;</span> <span class="hljs-keyword">for</span> v <span class="hljs-keyword">in</span> row]
        lines.append(<span class="hljs-string">&quot;|&quot;</span> + <span class="hljs-string">&quot; |&quot;</span>.join(cells) + <span class="hljs-string">&quot; |&quot;</span>)
        lines.append(<span class="hljs-string">&quot;+------&quot;</span> * <span class="hljs-built_in">len</span>(row) + <span class="hljs-string">&quot;+&quot;</span>)
    <span class="hljs-keyword">return</span> <span class="hljs-string">&quot;\\n&quot;</span>.join(lines)

<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_max_tile</span>(<span class="hljs-params">board: <span class="hljs-type">List</span>[<span class="hljs-type">List</span>[<span class="hljs-built_in">int</span>]]</span>) -&gt; <span class="hljs-built_in">int</span>:
    <span class="hljs-string">&quot;&quot;&quot;Get highest tile value.&quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">max</span>(cell <span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> board <span class="hljs-keyword">for</span> cell <span class="hljs-keyword">in</span> row)`,lang:"python",wrap:!1}}),D=new T({props:{title:"Part 4: Reward Function Design",local:"part-4-reward-function-design",headingTag:"h2"}}),O=new T({props:{title:"Reward Implementation",local:"reward-implementation",headingTag:"h3"}}),ll=new w({props:{code:"aW1wb3J0JTIwbWF0aCUwQSUwQWRlZiUyMGNhbGN1bGF0ZV9yZXdhcmQoJTBBJTIwJTIwJTIwJTIwbWF4X3RpbGUlM0ElMjBpbnQlMkMlMEElMjAlMjAlMjAlMjBzdWNjZXNzJTNBJTIwYm9vbCUyQyUwQSUyMCUyMCUyMCUyMGNvZGVfZXJyb3IlM0ElMjBib29sJTIwJTNEJTIwRmFsc2UlMEEpJTIwLSUzRSUyMGZsb2F0JTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwQ2FsY3VsYXRlJTIwcmV3YXJkJTIwZm9yJTIwYSUyMDIwNDglMjBnYW1lJTIwb3V0Y29tZS4lMEElMEElMjAlMjAlMjAlMjBBcmdzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWF4X3RpbGUlM0ElMjBIaWdoZXN0JTIwdGlsZSUyMGFjaGlldmVkJTIwKDIlMkMlMjA0JTJDJTIwOCUyQyUyMC4uLiUyQyUyMDIwNDgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VjY2VzcyUzQSUyMFdoZXRoZXIlMjB3ZSUyMHJlYWNoZWQlMjAyMDQ4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29kZV9lcnJvciUzQSUyMFdoZXRoZXIlMjBnZW5lcmF0ZWQlMjBjb2RlJTIwaGFkJTIwZXJyb3JzJTBBJTBBJTIwJTIwJTIwJTIwUmV0dXJucyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEZsb2F0JTIwcmV3YXJkJTIwdmFsdWUlMEElMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjBpZiUyMGNvZGVfZXJyb3IlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAtMC41JTIwJTIwJTIzJTIwUGVuYWx0eSUyMGZvciUyMGludmFsaWQlMjBjb2RlJTBBJTBBJTIwJTIwJTIwJTIwaWYlMjBzdWNjZXNzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwMS4wJTIwJTIwJTIzJTIwRnVsbCUyMHJld2FyZCUyMGZvciUyMHdpbm5pbmclMEElMEElMjAlMjAlMjAlMjAlMjMlMjBQcm9ncmVzcyUyMHJld2FyZCUzQSUyMGxvZyUyMHNjYWxlJTIwZnJvbSUyMDAlMjB0byUyMDAuOSUwQSUyMCUyMCUyMCUyMGlmJTIwbWF4X3RpbGUlMjAlM0UlMjAwJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJvZ3Jlc3MlMjAlM0QlMjBtYXRoLmxvZzIobWF4X3RpbGUpJTIwJTJGJTIwbWF0aC5sb2cyKDIwNDgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwbWluKDAuOSUyQyUyMHByb2dyZXNzKSUwQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMDAuMCUwQSUwQSUyMyUyMFRlc3QlMjByZXdhcmQlMjBmdW5jdGlvbiUwQXRlc3RfY2FzZXMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAoMjA0OCUyQyUyMFRydWUlMkMlMjBGYWxzZSUyQyUyMCUyMldvbiElMjIpJTJDJTBBJTIwJTIwJTIwJTIwKDEwMjQlMkMlMjBGYWxzZSUyQyUyMEZhbHNlJTJDJTIwJTIyR290JTIwdG8lMjAxMDI0JTIyKSUyQyUwQSUyMCUyMCUyMCUyMCg1MTIlMkMlMjBGYWxzZSUyQyUyMEZhbHNlJTJDJTIwJTIyR290JTIwdG8lMjA1MTIlMjIpJTJDJTBBJTIwJTIwJTIwJTIwKDY0JTJDJTIwRmFsc2UlMkMlMjBGYWxzZSUyQyUyMCUyMkVhcmx5JTIwZ2FtZSUyMiklMkMlMEElNUQlMEElMEFmb3IlMjBtYXhfdGlsZSUyQyUyMHN1Y2Nlc3MlMkMlMjBlcnJvciUyQyUyMGRlc2MlMjBpbiUyMHRlc3RfY2FzZXMlM0ElMEElMjAlMjAlMjAlMjByZXdhcmQlMjAlM0QlMjBjYWxjdWxhdGVfcmV3YXJkKG1heF90aWxlJTJDJTIwc3VjY2VzcyUyQyUyMGVycm9yKSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlN0JkZXNjJTNBMjBzJTdEJTIwLSUzRSUyMFJld2FyZCUzQSUyMCU3QnJld2FyZCUzQSUyQi4zZiU3RCUyMik=",highlighted:`<span class="hljs-keyword">import</span> math

<span class="hljs-keyword">def</span> <span class="hljs-title function_">calculate_reward</span>(<span class="hljs-params">
    max_tile: <span class="hljs-built_in">int</span>,
    success: <span class="hljs-built_in">bool</span>,
    code_error: <span class="hljs-built_in">bool</span> = <span class="hljs-literal">False</span>
</span>) -&gt; <span class="hljs-built_in">float</span>:
    <span class="hljs-string">&quot;&quot;&quot;
    Calculate reward for a 2048 game outcome.

    Args:
        max_tile: Highest tile achieved (2, 4, 8, ..., 2048)
        success: Whether we reached 2048
        code_error: Whether generated code had errors

    Returns:
        Float reward value
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">if</span> code_error:
        <span class="hljs-keyword">return</span> -<span class="hljs-number">0.5</span>  <span class="hljs-comment"># Penalty for invalid code</span>

    <span class="hljs-keyword">if</span> success:
        <span class="hljs-keyword">return</span> <span class="hljs-number">1.0</span>  <span class="hljs-comment"># Full reward for winning</span>

    <span class="hljs-comment"># Progress reward: log scale from 0 to 0.9</span>
    <span class="hljs-keyword">if</span> max_tile &gt; <span class="hljs-number">0</span>:
        progress = math.log2(max_tile) / math.log2(<span class="hljs-number">2048</span>)
        <span class="hljs-keyword">return</span> <span class="hljs-built_in">min</span>(<span class="hljs-number">0.9</span>, progress)

    <span class="hljs-keyword">return</span> <span class="hljs-number">0.0</span>

<span class="hljs-comment"># Test reward function</span>
test_cases = [
    (<span class="hljs-number">2048</span>, <span class="hljs-literal">True</span>, <span class="hljs-literal">False</span>, <span class="hljs-string">&quot;Won!&quot;</span>),
    (<span class="hljs-number">1024</span>, <span class="hljs-literal">False</span>, <span class="hljs-literal">False</span>, <span class="hljs-string">&quot;Got to 1024&quot;</span>),
    (<span class="hljs-number">512</span>, <span class="hljs-literal">False</span>, <span class="hljs-literal">False</span>, <span class="hljs-string">&quot;Got to 512&quot;</span>),
    (<span class="hljs-number">64</span>, <span class="hljs-literal">False</span>, <span class="hljs-literal">False</span>, <span class="hljs-string">&quot;Early game&quot;</span>),
]

<span class="hljs-keyword">for</span> max_tile, success, error, desc <span class="hljs-keyword">in</span> test_cases:
    reward = calculate_reward(max_tile, success, error)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{desc:20s}</span> -&gt; Reward: <span class="hljs-subst">{reward:+<span class="hljs-number">.3</span>f}</span>&quot;</span>)`,lang:"python",wrap:!1}}),sl=new T({props:{title:"Part 5: Strategy Generation",local:"part-5-strategy-generation",headingTag:"h2"}}),nl=new T({props:{title:"Prompt Template",local:"prompt-template",headingTag:"h3"}}),tl=new w({props:{code:"U1lTVEVNX1BST01QVCUyMCUzRCUyMCUyMiUyMiUyMllvdSUyMGFyZSUyMGFuJTIwZXhwZXJ0JTIwYXQlMjBwbGF5aW5nJTIwMjA0OC4lMjBHZW5lcmF0ZSUyMGElMjBQeXRob24lMjBmdW5jdGlvbiUwQXRoYXQlMjB0YWtlcyUyMGElMjBib2FyZCUyMHN0YXRlJTIwYW5kJTIwcmV0dXJucyUyMHRoZSUyMGJlc3QlMjBhY3Rpb24lMjAoMCUzRFVQJTJDJTIwMSUzRFJJR0hUJTJDJTIwMiUzRERPV04lMkMlMjAzJTNETEVGVCkuJTBBJTBBVGhlJTIwYm9hcmQlMjBpcyUyMGElMjA0eDQlMjBsaXN0JTIwb2YlMjBpbnRlZ2Vycy4lMjBFbXB0eSUyMGNlbGxzJTIwYXJlJTIwMC4lMEFZb3VyJTIwZnVuY3Rpb24lMjBzaG91bGQlMjBhbmFseXplJTIwdGhlJTIwYm9hcmQlMjBhbmQlMjByZXR1cm4lMjBhbiUyMG9wdGltYWwlMjBtb3ZlLiUwQSUyMiUyMiUyMiUwQSUwQWRlZiUyMGNyZWF0ZV9wcm9tcHQoYm9hcmQlM0ElMjBMaXN0JTVCTGlzdCU1QmludCU1RCU1RCklMjAtJTNFJTIwc3RyJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyQ3JlYXRlJTIwcHJvbXB0JTIwZm9yJTIwc3RyYXRlZ3klMjBnZW5lcmF0aW9uLiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMGJvYXJkX3N0ciUyMCUzRCUyMCUyMiU1Q24lMjIuam9pbihzdHIocm93KSUyMGZvciUyMHJvdyUyMGluJTIwYm9hcmQpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwZiUyMiUyMiUyMiU3QlNZU1RFTV9QUk9NUFQlN0QlMEElMEFDdXJyZW50JTIwYm9hcmQlM0ElMEElN0Jib2FyZF9zdHIlN0QlMEElMEFHZW5lcmF0ZSUyMGElMjBzdHJhdGVneSUyMGZ1bmN0aW9uJTNBJTBBJTYwJTYwJTYwcHl0aG9uJTBBZGVmJTIwc3RyYXRlZ3koYm9hcmQpJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwWW91ciUyMGNvZGUlMjBoZXJlJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwYWN0aW9uJTIwJTIwJTIzJTIwMCUyQyUyMDElMkMlMjAyJTJDJTIwb3IlMjAzJTBBJTYwJTYwJTYwJTIyJTIyJTIy",highlighted:`SYSTEM_PROMPT = <span class="hljs-string">&quot;&quot;&quot;You are an expert at playing 2048. Generate a Python function
that takes a board state and returns the best action (0=UP, 1=RIGHT, 2=DOWN, 3=LEFT).

The board is a 4x4 list of integers. Empty cells are 0.
Your function should analyze the board and return an optimal move.
&quot;&quot;&quot;</span>

<span class="hljs-keyword">def</span> <span class="hljs-title function_">create_prompt</span>(<span class="hljs-params">board: <span class="hljs-type">List</span>[<span class="hljs-type">List</span>[<span class="hljs-built_in">int</span>]]</span>) -&gt; <span class="hljs-built_in">str</span>:
    <span class="hljs-string">&quot;&quot;&quot;Create prompt for strategy generation.&quot;&quot;&quot;</span>
    board_str = <span class="hljs-string">&quot;\\n&quot;</span>.join(<span class="hljs-built_in">str</span>(row) <span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> board)
    <span class="hljs-keyword">return</span> <span class="hljs-string">f&quot;&quot;&quot;<span class="hljs-subst">{SYSTEM_PROMPT}</span>

Current board:
<span class="hljs-subst">{board_str}</span>

Generate a strategy function:
\`\`\`python
def strategy(board):
    # Your code here
    return action  # 0, 1, 2, or 3
\`\`\`&quot;&quot;&quot;</span>`,lang:"python",wrap:!1}}),el=new T({props:{title:"Executing Generated Strategies",local:"executing-generated-strategies",headingTag:"h3"}}),Ml=new w({props:{code:"aW1wb3J0JTIwYXN0JTBBZnJvbSUyMHR5cGluZyUyMGltcG9ydCUyMENhbGxhYmxlJTBBJTBBZGVmJTIwZXh0cmFjdF9hbmRfZXhlY3V0ZV9zdHJhdGVneSglMEElMjAlMjAlMjAlMjBnZW5lcmF0ZWRfY29kZSUzQSUyMHN0ciUyQyUwQSUyMCUyMCUyMCUyMGJvYXJkJTNBJTIwTGlzdCU1Qkxpc3QlNUJpbnQlNUQlNUQlMkMlMEElMjAlMjAlMjAlMjB0aW1lb3V0JTNBJTIwZmxvYXQlMjAlM0QlMjA1LjAlMEEpJTIwLSUzRSUyMHR1cGxlJTVCaW50JTJDJTIwYm9vbCU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMEV4dHJhY3QlMjBhbmQlMjBleGVjdXRlJTIwYSUyMGdlbmVyYXRlZCUyMHN0cmF0ZWd5JTIwZnVuY3Rpb24uJTBBJTBBJTIwJTIwJTIwJTIwUmV0dXJucyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMChhY3Rpb24lMkMlMjBzdWNjZXNzKSUzQSUyMFRoZSUyMGFjdGlvbiUyMHRvJTIwdGFrZSUyMGFuZCUyMHdoZXRoZXIlMjBleGVjdXRpb24lMjBzdWNjZWVkZWQlMEElMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBFeHRyYWN0JTIwY29kZSUyMGJsb2NrJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjAlMjIlNjAlNjAlNjBweXRob24lMjIlMjBpbiUyMGdlbmVyYXRlZF9jb2RlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29kZSUyMCUzRCUyMGdlbmVyYXRlZF9jb2RlLnNwbGl0KCUyMiU2MCU2MCU2MHB5dGhvbiUyMiklNUIxJTVELnNwbGl0KCUyMiU2MCU2MCU2MCUyMiklNUIwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvZGUlMjAlM0QlMjBnZW5lcmF0ZWRfY29kZSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFBhcnNlJTIwYW5kJTIwdmFsaWRhdGUlMjBBU1QlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cmVlJTIwJTNEJTIwYXN0LnBhcnNlKGNvZGUpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwRXhlY3V0ZSUyMGluJTIwc2FuZGJveCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG5hbWVzcGFjZSUyMCUzRCUyMCU3QiUyMmJvYXJkJTIyJTNBJTIwYm9hcmQlN0QlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBleGVjKGNvbXBpbGUodHJlZSUyQyUyMCUyMiUzQ3N0cmF0ZWd5JTNFJTIyJTJDJTIwJTIyZXhlYyUyMiklMkMlMjBuYW1lc3BhY2UpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwQ2FsbCUyMHRoZSUyMHN0cmF0ZWd5JTIwZnVuY3Rpb24lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMCUyMnN0cmF0ZWd5JTIyJTIwaW4lMjBuYW1lc3BhY2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhY3Rpb24lMjAlM0QlMjBuYW1lc3BhY2UlNUIlMjJzdHJhdGVneSUyMiU1RChib2FyZCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGFjdGlvbiUyMGluJTIwJTVCMCUyQyUyMDElMkMlMjAyJTJDJTIwMyU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMGFjdGlvbiUyQyUyMFRydWUlMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAwJTJDJTIwRmFsc2UlMjAlMjAlMjMlMjBEZWZhdWx0JTIwYWN0aW9uJTIwb24lMjBmYWlsdXJlJTBBJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwRXhjZXB0aW9uJTIwYXMlMjBlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZiUyMlN0cmF0ZWd5JTIwZXhlY3V0aW9uJTIwZXJyb3IlM0ElMjAlN0JlJTdEJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMDAlMkMlMjBGYWxzZQ==",highlighted:`<span class="hljs-keyword">import</span> ast
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">Callable</span>

<span class="hljs-keyword">def</span> <span class="hljs-title function_">extract_and_execute_strategy</span>(<span class="hljs-params">
    generated_code: <span class="hljs-built_in">str</span>,
    board: <span class="hljs-type">List</span>[<span class="hljs-type">List</span>[<span class="hljs-built_in">int</span>]],
    timeout: <span class="hljs-built_in">float</span> = <span class="hljs-number">5.0</span>
</span>) -&gt; <span class="hljs-built_in">tuple</span>[<span class="hljs-built_in">int</span>, <span class="hljs-built_in">bool</span>]:
    <span class="hljs-string">&quot;&quot;&quot;
    Extract and execute a generated strategy function.

    Returns:
        (action, success): The action to take and whether execution succeeded
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">try</span>:
        <span class="hljs-comment"># Extract code block</span>
        <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;\`\`\`python&quot;</span> <span class="hljs-keyword">in</span> generated_code:
            code = generated_code.split(<span class="hljs-string">&quot;\`\`\`python&quot;</span>)[<span class="hljs-number">1</span>].split(<span class="hljs-string">&quot;\`\`\`&quot;</span>)[<span class="hljs-number">0</span>]
        <span class="hljs-keyword">else</span>:
            code = generated_code

        <span class="hljs-comment"># Parse and validate AST</span>
        tree = ast.parse(code)

        <span class="hljs-comment"># Execute in sandbox</span>
        namespace = {<span class="hljs-string">&quot;board&quot;</span>: board}
        <span class="hljs-built_in">exec</span>(<span class="hljs-built_in">compile</span>(tree, <span class="hljs-string">&quot;&lt;strategy&gt;&quot;</span>, <span class="hljs-string">&quot;exec&quot;</span>), namespace)

        <span class="hljs-comment"># Call the strategy function</span>
        <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;strategy&quot;</span> <span class="hljs-keyword">in</span> namespace:
            action = namespace[<span class="hljs-string">&quot;strategy&quot;</span>](board)
            <span class="hljs-keyword">if</span> action <span class="hljs-keyword">in</span> [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>]:
                <span class="hljs-keyword">return</span> action, <span class="hljs-literal">True</span>

        <span class="hljs-keyword">return</span> <span class="hljs-number">0</span>, <span class="hljs-literal">False</span>  <span class="hljs-comment"># Default action on failure</span>

    <span class="hljs-keyword">except</span> Exception <span class="hljs-keyword">as</span> e:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Strategy execution error: <span class="hljs-subst">{e}</span>&quot;</span>)
        <span class="hljs-keyword">return</span> <span class="hljs-number">0</span>, <span class="hljs-literal">False</span>`,lang:"python",wrap:!1}}),pl=new T({props:{title:"Part 6: GRPO Training",local:"part-6-grpo-training",headingTag:"h2"}}),il=new T({props:{title:"Training Configuration",local:"training-configuration",headingTag:"h3"}}),Jl=new w({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMkMlMjBHUlBPVHJhaW5lciUwQSUwQWdycG9fY29uZmlnJTIwJTNEJTIwR1JQT0NvbmZpZyglMEElMjAlMjAlMjAlMjAlMjMlMjBMZWFybmluZyUyMHJhdGUlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNiUyQyUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMEJhdGNoJTIwc2l6ZXMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNENCUyQyUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMFRyYWluaW5nJTIwZHVyYXRpb24lMEElMjAlMjAlMjAlMjBtYXhfc3RlcHMlM0QyMDAlMkMlMEElMEElMjAlMjAlMjAlMjAlMjMlMjBNZW1vcnklMjBvcHRpbWl6YXRpb24lMEElMjAlMjAlMjAlMjBiZjE2JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2NoZWNrcG9pbnRpbmclM0RUcnVlJTJDJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwTG9nZ2luZyUwQSUyMCUyMCUyMCUyMGxvZ2dpbmdfc3RlcHMlM0QxJTJDJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMi4lMkYyMDQ4X2dycG9fb3V0cHV0JTIyJTJDJTBBJTIwJTIwJTIwJTIwcmVwb3J0X3RvJTNEJTIybm9uZSUyMiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig, GRPOTrainer

grpo_config = GRPOConfig(
    <span class="hljs-comment"># Learning rate</span>
    learning_rate=<span class="hljs-number">2e-6</span>,

    <span class="hljs-comment"># Batch sizes</span>
    per_device_train_batch_size=<span class="hljs-number">4</span>,
    gradient_accumulation_steps=<span class="hljs-number">4</span>,

    <span class="hljs-comment"># Training duration</span>
    max_steps=<span class="hljs-number">200</span>,

    <span class="hljs-comment"># Memory optimization</span>
    bf16=<span class="hljs-literal">True</span>,
    gradient_checkpointing=<span class="hljs-literal">True</span>,

    <span class="hljs-comment"># Logging</span>
    logging_steps=<span class="hljs-number">1</span>,
    output_dir=<span class="hljs-string">&quot;./2048_grpo_output&quot;</span>,
    report_to=<span class="hljs-string">&quot;none&quot;</span>,
)`,lang:"python",wrap:!1}}),Ul=new T({props:{title:"Training Loop",local:"training-loop",headingTag:"h3"}}),Tl=new w({props:{code:"ZGVmJTIwdHJhaW5fMjA0OF9hZ2VudCglMEElMjAlMjAlMjAlMjBtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMGVudiUyQyUwQSUyMCUyMCUyMCUyMGNvbmZpZyUzQSUyMEdSUE9Db25maWclMkMlMEElMjAlMjAlMjAlMjBudW1fZXBpc29kZXMlM0ElMjBpbnQlMjAlM0QlMjAxMDAlMkMlMEEpJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwVHJhaW4lMjB0aGUlMjBtb2RlbCUyMHRvJTIwcGxheSUyMDIwNDglMjB1c2luZyUyMEdSUE8uJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIzJTIwUHJlcGFyZSUyMG1vZGVsJTIwZm9yJTIwdHJhaW5pbmclMEElMjAlMjAlMjAlMjBGYXN0TGFuZ3VhZ2VNb2RlbC5mb3JfdHJhaW5pbmcobW9kZWwpJTBBJTBBJTIwJTIwJTIwJTIwdHJhaW5pbmdfZGF0YSUyMCUzRCUyMCU1QiU1RCUwQSUwQSUyMCUyMCUyMCUyMGZvciUyMGVwaXNvZGUlMjBpbiUyMHJhbmdlKG51bV9lcGlzb2RlcyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBSZXNldCUyMGVudmlyb25tZW50JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwZW52LnJlc2V0KCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBib2FyZCUyMCUzRCUyMGluZm9fc3RhdGVfdG9fYm9hcmQocmVzdWx0Lm9ic2VydmF0aW9uLmluZm9fc3RhdGUpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXBpc29kZV9yZXdhcmQlMjAlM0QlMjAwJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RlcHMlMjAlM0QlMjAwJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2hpbGUlMjBub3QlMjByZXN1bHQuZG9uZSUyMGFuZCUyMHN0ZXBzJTIwJTNDJTIwMTAwMCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEdlbmVyYXRlJTIwc3RyYXRlZ3klMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcm9tcHQlMjAlM0QlMjBjcmVhdGVfcHJvbXB0KGJvYXJkKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCoqaW5wdXRzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWF4X25ld190b2tlbnMlM0QyNTYlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzRDAuNyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRvX3NhbXBsZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ2VuZXJhdGVkJTIwJTNEJTIwdG9rZW5pemVyLmRlY29kZShvdXRwdXRzJTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEV4ZWN1dGUlMjBzdHJhdGVneSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFjdGlvbiUyQyUyMHN1Y2Nlc3MlMjAlM0QlMjBleHRyYWN0X2FuZF9leGVjdXRlX3N0cmF0ZWd5KGdlbmVyYXRlZCUyQyUyMGJvYXJkKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFRha2UlMjBhY3Rpb24lMjBpbiUyMGVudmlyb25tZW50JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW52X2FjdGlvbiUyMCUzRCUyME9wZW5TcGllbEFjdGlvbihhY3Rpb25faWQlM0RhY3Rpb24lMkMlMjBnYW1lX25hbWUlM0QlMjIyMDQ4JTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlc3VsdCUyMCUzRCUyMGVudi5zdGVwKGVudl9hY3Rpb24pJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwVXBkYXRlJTIwYm9hcmQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBib2FyZCUyMCUzRCUyMGluZm9fc3RhdGVfdG9fYm9hcmQocmVzdWx0Lm9ic2VydmF0aW9uLmluZm9fc3RhdGUpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXBpc29kZV9yZXdhcmQlMjAlMkIlM0QlMjByZXN1bHQucmV3YXJkJTIwaWYlMjByZXN1bHQucmV3YXJkJTIwZWxzZSUyMDAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdGVwcyUyMCUyQiUzRCUyMDElMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBDYWxjdWxhdGUlMjBmaW5hbCUyMHJld2FyZCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1heF90aWxlJTIwJTNEJTIwZ2V0X21heF90aWxlKGJvYXJkKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZpbmFsX3Jld2FyZCUyMCUzRCUyMGNhbGN1bGF0ZV9yZXdhcmQobWF4X3RpbGUlMkMlMjBtYXhfdGlsZSUyMCUzRSUzRCUyMDIwNDgpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwU3RvcmUlMjBmb3IlMjB0cmFpbmluZyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRyYWluaW5nX2RhdGEuYXBwZW5kKCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnByb21wdCUyMiUzQSUyMHByb21wdCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJlc3BvbnNlJTIyJTNBJTIwZ2VuZXJhdGVkJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycmV3YXJkJTIyJTNBJTIwZmluYWxfcmV3YXJkJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwZXBpc29kZSUyMCUyNSUyMDEwJTIwJTNEJTNEJTIwMCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGYlMjJFcGlzb2RlJTIwJTdCZXBpc29kZSU3RCUzQSUyME1heCUyMHRpbGUlM0QlN0JtYXhfdGlsZSU3RCUyQyUyMFJld2FyZCUzRCU3QmZpbmFsX3Jld2FyZCUzQS4zZiU3RCUyMiklMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjB0cmFpbmluZ19kYXRh",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">train_2048_agent</span>(<span class="hljs-params">
    model,
    tokenizer,
    env,
    config: GRPOConfig,
    num_episodes: <span class="hljs-built_in">int</span> = <span class="hljs-number">100</span>,
</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Train the model to play 2048 using GRPO.
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># Prepare model for training</span>
    FastLanguageModel.for_training(model)

    training_data = []

    <span class="hljs-keyword">for</span> episode <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_episodes):
        <span class="hljs-comment"># Reset environment</span>
        result = env.reset()
        board = info_state_to_board(result.observation.info_state)

        episode_reward = <span class="hljs-number">0</span>
        steps = <span class="hljs-number">0</span>

        <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> result.done <span class="hljs-keyword">and</span> steps &lt; <span class="hljs-number">1000</span>:
            <span class="hljs-comment"># Generate strategy</span>
            prompt = create_prompt(board)
            inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

            outputs = model.generate(
                **inputs,
                max_new_tokens=<span class="hljs-number">256</span>,
                temperature=<span class="hljs-number">0.7</span>,
                do_sample=<span class="hljs-literal">True</span>,
            )

            generated = tokenizer.decode(outputs[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>)

            <span class="hljs-comment"># Execute strategy</span>
            action, success = extract_and_execute_strategy(generated, board)

            <span class="hljs-comment"># Take action in environment</span>
            env_action = OpenSpielAction(action_id=action, game_name=<span class="hljs-string">&quot;2048&quot;</span>)
            result = env.step(env_action)

            <span class="hljs-comment"># Update board</span>
            board = info_state_to_board(result.observation.info_state)
            episode_reward += result.reward <span class="hljs-keyword">if</span> result.reward <span class="hljs-keyword">else</span> <span class="hljs-number">0</span>
            steps += <span class="hljs-number">1</span>

        <span class="hljs-comment"># Calculate final reward</span>
        max_tile = get_max_tile(board)
        final_reward = calculate_reward(max_tile, max_tile &gt;= <span class="hljs-number">2048</span>)

        <span class="hljs-comment"># Store for training</span>
        training_data.append({
            <span class="hljs-string">&quot;prompt&quot;</span>: prompt,
            <span class="hljs-string">&quot;response&quot;</span>: generated,
            <span class="hljs-string">&quot;reward&quot;</span>: final_reward,
        })

        <span class="hljs-keyword">if</span> episode % <span class="hljs-number">10</span> == <span class="hljs-number">0</span>:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Episode <span class="hljs-subst">{episode}</span>: Max tile=<span class="hljs-subst">{max_tile}</span>, Reward=<span class="hljs-subst">{final_reward:<span class="hljs-number">.3</span>f}</span>&quot;</span>)

    <span class="hljs-keyword">return</span> training_data`,lang:"python",wrap:!1}}),rl=new T({props:{title:"Part 7: Deployment",local:"part-7-deployment",headingTag:"h2"}}),ol=new T({props:{title:"Saving the Model",local:"saving-the-model",headingTag:"h3"}}),wl=new w({props:{code:"JTIzJTIwU2F2ZSUyMExvUkElMjBhZGFwdGVycyUyMG9ubHklMEFtb2RlbC5zYXZlX3ByZXRyYWluZWQoJTIyLiUyRjIwNDhfc3RyYXRlZ3lfbW9kZWwlMjIpJTBBdG9rZW5pemVyLnNhdmVfcHJldHJhaW5lZCglMjIuJTJGMjA0OF9zdHJhdGVneV9tb2RlbCUyMiklMEElMEElMjMlMjBTYXZlJTIwbWVyZ2VkJTIwbW9kZWwlMjBmb3IlMjBpbmZlcmVuY2UlMEFtb2RlbC5zYXZlX3ByZXRyYWluZWRfbWVyZ2VkKCUwQSUyMCUyMCUyMCUyMCUyMi4lMkYyMDQ4X3N0cmF0ZWd5X21vZGVsX21lcmdlZCUyMiUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfbWV0aG9kJTNEJTIybWVyZ2VkXzE2Yml0JTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-comment"># Save LoRA adapters only</span>
model.save_pretrained(<span class="hljs-string">&quot;./2048_strategy_model&quot;</span>)
tokenizer.save_pretrained(<span class="hljs-string">&quot;./2048_strategy_model&quot;</span>)

<span class="hljs-comment"># Save merged model for inference</span>
model.save_pretrained_merged(
    <span class="hljs-string">&quot;./2048_strategy_model_merged&quot;</span>,
    tokenizer,
    save_method=<span class="hljs-string">&quot;merged_16bit&quot;</span>,
)`,lang:"python",wrap:!1}}),cl=new T({props:{title:"Push to Hugging Face Hub",local:"push-to-hugging-face-hub",headingTag:"h3"}}),ml=new w({props:{code:"JTIzJTIwUHVzaCUyMHRvJTIwSHViJTBBbW9kZWwucHVzaF90b19odWIoJTBBJTIwJTIwJTIwJTIwJTIyeW91ci11c2VybmFtZSUyRjIwNDgtc3RyYXRlZ3ktbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBzYXZlX21ldGhvZCUzRCUyMm1lcmdlZF8xNmJpdCUyMiUyQyUwQSUyMCUyMCUyMCUyMHByaXZhdGUlM0RGYWxzZSUyQyUwQSklMEElMEFwcmludCglMjJNb2RlbCUyMGRlcGxveWVkJTIwdG8lM0ElMjBodWdnaW5nZmFjZS5jbyUyRnlvdXItdXNlcm5hbWUlMkYyMDQ4LXN0cmF0ZWd5LW1vZGVsJTIyKQ==",highlighted:`<span class="hljs-comment"># Push to Hub</span>
model.push_to_hub(
    <span class="hljs-string">&quot;your-username/2048-strategy-model&quot;</span>,
    tokenizer,
    save_method=<span class="hljs-string">&quot;merged_16bit&quot;</span>,
    private=<span class="hljs-literal">False</span>,
)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Model deployed to: huggingface.co/your-username/2048-strategy-model&quot;</span>)`,lang:"python",wrap:!1}}),ul=new T({props:{title:"Using the Trained Model",local:"using-the-trained-model",headingTag:"h3"}}),Il=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQSUyMyUyMExvYWQlMjB0cmFpbmVkJTIwbW9kZWwlMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJ5b3VyLXVzZXJuYW1lJTJGMjA0OC1zdHJhdGVneS1tb2RlbCUyMiklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJ5b3VyLXVzZXJuYW1lJTJGMjA0OC1zdHJhdGVneS1tb2RlbCUyMiklMEElMEElMjMlMjBHZW5lcmF0ZSUyMHN0cmF0ZWd5JTBBZGVmJTIwZ2V0X2FjdGlvbihib2FyZCUzQSUyMExpc3QlNUJMaXN0JTVCaW50JTVEJTVEKSUyMC0lM0UlMjBpbnQlM0ElMEElMjAlMjAlMjAlMjBwcm9tcHQlMjAlM0QlMjBjcmVhdGVfcHJvbXB0KGJvYXJkKSUwQSUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjU2KSUwQSUyMCUyMCUyMCUyMGdlbmVyYXRlZCUyMCUzRCUyMHRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklMEElMjAlMjAlMjAlMjBhY3Rpb24lMkMlMjBfJTIwJTNEJTIwZXh0cmFjdF9hbmRfZXhlY3V0ZV9zdHJhdGVneShnZW5lcmF0ZWQlMkMlMjBib2FyZCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBhY3Rpb24lMEElMEElMjMlMjBQbGF5JTIwYSUyMGdhbWUlMEF3aXRoJTIwT3BlblNwaWVsRW52LmZyb21faHViKCUyMm9wZW5lbnYlMkZvcGVuc3BpZWwtZW52JTIyKSUyMGFzJTIwZW52JTNBJTBBJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwZW52LnJlc2V0KCklMEElMjAlMjAlMjAlMjBib2FyZCUyMCUzRCUyMGluZm9fc3RhdGVfdG9fYm9hcmQocmVzdWx0Lm9ic2VydmF0aW9uLmluZm9fc3RhdGUpJTBBJTBBJTIwJTIwJTIwJTIwd2hpbGUlMjBub3QlMjByZXN1bHQuZG9uZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFjdGlvbiUyMCUzRCUyMGdldF9hY3Rpb24oYm9hcmQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwZW52LnN0ZXAoT3BlblNwaWVsQWN0aW9uKGFjdGlvbl9pZCUzRGFjdGlvbiUyQyUyMGdhbWVfbmFtZSUzRCUyMjIwNDglMjIpKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGJvYXJkJTIwJTNEJTIwaW5mb19zdGF0ZV90b19ib2FyZChyZXN1bHQub2JzZXJ2YXRpb24uaW5mb19zdGF0ZSklMEElMEElMjAlMjAlMjAlMjBwcmludChmJTIyRmluYWwlMjBtYXglMjB0aWxlJTNBJTIwJTdCZ2V0X21heF90aWxlKGJvYXJkKSU3RCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

<span class="hljs-comment"># Load trained model</span>
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;your-username/2048-strategy-model&quot;</span>)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;your-username/2048-strategy-model&quot;</span>)

<span class="hljs-comment"># Generate strategy</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_action</span>(<span class="hljs-params">board: <span class="hljs-type">List</span>[<span class="hljs-type">List</span>[<span class="hljs-built_in">int</span>]]</span>) -&gt; <span class="hljs-built_in">int</span>:
    prompt = create_prompt(board)
    inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
    outputs = model.generate(**inputs, max_new_tokens=<span class="hljs-number">256</span>)
    generated = tokenizer.decode(outputs[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>)
    action, _ = extract_and_execute_strategy(generated, board)
    <span class="hljs-keyword">return</span> action

<span class="hljs-comment"># Play a game</span>
<span class="hljs-keyword">with</span> OpenSpielEnv.from_hub(<span class="hljs-string">&quot;openenv/openspiel-env&quot;</span>) <span class="hljs-keyword">as</span> env:
    result = env.reset()
    board = info_state_to_board(result.observation.info_state)

    <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> result.done:
        action = get_action(board)
        result = env.step(OpenSpielAction(action_id=action, game_name=<span class="hljs-string">&quot;2048&quot;</span>))
        board = info_state_to_board(result.observation.info_state)

    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Final max tile: <span class="hljs-subst">{get_max_tile(board)}</span>&quot;</span>)`,lang:"python",wrap:!1}}),dl=new T({props:{title:"Preventing Reward Hacking",local:"preventing-reward-hacking",headingTag:"h2"}}),bl=new w({props:{code:"aW1wb3J0JTIwcmVzb3VyY2UlMEFpbXBvcnQlMjBzaWduYWwlMEElMEFkZWYlMjBzYWZlX2V4ZWN1dGUoY29kZSUzQSUyMHN0ciUyQyUyMGJvYXJkJTNBJTIwTGlzdCU1Qkxpc3QlNUJpbnQlNUQlNUQlMkMlMjB0aW1lb3V0JTNBJTIwZmxvYXQlMjAlM0QlMjA1LjApJTIwLSUzRSUyMGludCUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMkV4ZWN1dGUlMjBzdHJhdGVneSUyMHdpdGglMjBzYWZldHklMjBsaW1pdHMuJTIyJTIyJTIyJTBBJTBBJTIwJTIwJTIwJTIwZGVmJTIwaGFuZGxlcihzaWdudW0lMkMlMjBmcmFtZSklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByYWlzZSUyMFRpbWVvdXRFcnJvciglMjJTdHJhdGVneSUyMHRpbWVkJTIwb3V0JTIyKSUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMFNldCUyMHRpbWVvdXQlMEElMjAlMjAlMjAlMjBzaWduYWwuc2lnbmFsKHNpZ25hbC5TSUdBTFJNJTJDJTIwaGFuZGxlciklMEElMjAlMjAlMjAlMjBzaWduYWwuYWxhcm0oaW50KHRpbWVvdXQpKSUwQSUwQSUyMCUyMCUyMCUyMHRyeSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFNldCUyMG1lbW9yeSUyMGxpbWl0JTIwKDEwME1CKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlc291cmNlLnNldHJsaW1pdChyZXNvdXJjZS5STElNSVRfQVMlMkMlMjAoMTAwJTIwKiUyMDEwMjQlMjAqJTIwMTAyNCUyQyUyMC0xKSklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBFeGVjdXRlJTIwaW4lMjByZXN0cmljdGVkJTIwbmFtZXNwYWNlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbmFtZXNwYWNlJTIwJTNEJTIwJTdCJTIyYm9hcmQlMjIlM0ElMjBib2FyZCUyQyUyMCUyMl9fYnVpbHRpbnNfXyUyMiUzQSUyMCU3QiUyMmxlbiUyMiUzQSUyMGxlbiUyQyUyMCUyMm1heCUyMiUzQSUyMG1heCUyQyUyMCUyMm1pbiUyMiUzQSUyMG1pbiU3RCU3RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV4ZWMoY29kZSUyQyUyMG5hbWVzcGFjZSklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBuYW1lc3BhY2UuZ2V0KCUyMnN0cmF0ZWd5JTIyJTJDJTIwbGFtYmRhJTIwYiUzQSUyMDApKGJvYXJkKSUwQSUyMCUyMCUyMCUyMGZpbmFsbHklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzaWduYWwuYWxhcm0oMCk=",highlighted:`<span class="hljs-keyword">import</span> resource
<span class="hljs-keyword">import</span> signal

<span class="hljs-keyword">def</span> <span class="hljs-title function_">safe_execute</span>(<span class="hljs-params">code: <span class="hljs-built_in">str</span>, board: <span class="hljs-type">List</span>[<span class="hljs-type">List</span>[<span class="hljs-built_in">int</span>]], timeout: <span class="hljs-built_in">float</span> = <span class="hljs-number">5.0</span></span>) -&gt; <span class="hljs-built_in">int</span>:
    <span class="hljs-string">&quot;&quot;&quot;Execute strategy with safety limits.&quot;&quot;&quot;</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">handler</span>(<span class="hljs-params">signum, frame</span>):
        <span class="hljs-keyword">raise</span> TimeoutError(<span class="hljs-string">&quot;Strategy timed out&quot;</span>)

    <span class="hljs-comment"># Set timeout</span>
    signal.signal(signal.SIGALRM, handler)
    signal.alarm(<span class="hljs-built_in">int</span>(timeout))

    <span class="hljs-keyword">try</span>:
        <span class="hljs-comment"># Set memory limit (100MB)</span>
        resource.setrlimit(resource.RLIMIT_AS, (<span class="hljs-number">100</span> * <span class="hljs-number">1024</span> * <span class="hljs-number">1024</span>, -<span class="hljs-number">1</span>))

        <span class="hljs-comment"># Execute in restricted namespace</span>
        namespace = {<span class="hljs-string">&quot;board&quot;</span>: board, <span class="hljs-string">&quot;__builtins__&quot;</span>: {<span class="hljs-string">&quot;len&quot;</span>: <span class="hljs-built_in">len</span>, <span class="hljs-string">&quot;max&quot;</span>: <span class="hljs-built_in">max</span>, <span class="hljs-string">&quot;min&quot;</span>: <span class="hljs-built_in">min</span>}}
        <span class="hljs-built_in">exec</span>(code, namespace)

        <span class="hljs-keyword">return</span> namespace.get(<span class="hljs-string">&quot;strategy&quot;</span>, <span class="hljs-keyword">lambda</span> b: <span class="hljs-number">0</span>)(board)
    <span class="hljs-keyword">finally</span>:
        signal.alarm(<span class="hljs-number">0</span>)`,lang:"python",wrap:!1}}),gl=new T({props:{title:"Summary",local:"summary",headingTag:"h2"}}),Zl=new T({props:{title:"Next Steps",local:"next-steps",headingTag:"h2"}}),Al=new T({props:{title:"Related Resources",local:"related-resources",headingTag:"h2"}}),Ql=new Wa({props:{source:"https://github.com/huggingface/openenv/blob/main/docs/source/tutorials/rl-training-2048.md"}}),{c(){c=r("meta"),kl=t(),Wl=r("p"),$l=t(),M(u.$$.fragment),Sl=t(),M(I.$$.fragment),vl=t(),d=r("p"),d.textContent=sa,Vl=t(),m=r("blockquote"),m.innerHTML=aa,_l=t(),M(h.$$.fragment),Xl=t(),C=r("ul"),C.innerHTML=na,Nl=t(),M(b.$$.fragment),Fl=t(),g=r("p"),g.innerHTML=ta,Yl=t(),B=r("ul"),B.innerHTML=ea,zl=t(),f=r("p"),f.textContent=Ma,ql=t(),Z=r("ul"),Z.innerHTML=pa,Hl=t(),M(G.$$.fragment),xl=t(),M(A.$$.fragment),Ll=t(),M(E.$$.fragment),Dl=t(),M(Q.$$.fragment),Kl=t(),M(W.$$.fragment),Pl=t(),M(R.$$.fragment),Ol=t(),k=r("p"),k.textContent=ya,ls=t(),M($.$$.fragment),ss=t(),M(S.$$.fragment),as=t(),M(v.$$.fragment),ns=t(),M(V.$$.fragment),ts=t(),M(_.$$.fragment),es=t(),M(X.$$.fragment),Ms=t(),N=r("p"),N.textContent=ia,ps=t(),F=r("p"),F.innerHTML=Ja,ys=t(),Y=r("ul"),Y.innerHTML=Ua,is=t(),z=r("p"),z.innerHTML=Ta,Js=t(),M(q.$$.fragment),Us=t(),M(H.$$.fragment),Ts=t(),M(x.$$.fragment),rs=t(),M(L.$$.fragment),js=t(),M(D.$$.fragment),os=t(),K=r("p"),K.textContent=ra,ws=t(),P=r("ol"),P.innerHTML=ja,cs=t(),M(O.$$.fragment),ms=t(),M(ll.$$.fragment),us=t(),M(sl.$$.fragment),Is=t(),al=r("p"),al.textContent=oa,ds=t(),M(nl.$$.fragment),hs=t(),M(tl.$$.fragment),Cs=t(),M(el.$$.fragment),bs=t(),M(Ml.$$.fragment),gs=t(),M(pl.$$.fragment),Bs=t(),yl=r("p"),yl.textContent=wa,fs=t(),M(il.$$.fragment),Zs=t(),M(Jl.$$.fragment),Gs=t(),M(Ul.$$.fragment),As=t(),M(Tl.$$.fragment),Es=t(),M(rl.$$.fragment),Qs=t(),jl=r("p"),jl.textContent=ca,Ws=t(),M(ol.$$.fragment),Rs=t(),M(wl.$$.fragment),ks=t(),M(cl.$$.fragment),$s=t(),M(ml.$$.fragment),Ss=t(),M(ul.$$.fragment),vs=t(),M(Il.$$.fragment),Vs=t(),M(dl.$$.fragment),_s=t(),hl=r("p"),hl.textContent=ma,Xs=t(),Cl=r("ol"),Cl.innerHTML=ua,Ns=t(),M(bl.$$.fragment),Fs=t(),M(gl.$$.fragment),Ys=t(),Bl=r("p"),Bl.textContent=Ia,zs=t(),fl=r("ol"),fl.innerHTML=da,qs=t(),M(Zl.$$.fragment),Hs=t(),Gl=r("ul"),Gl.innerHTML=ha,xs=t(),M(Al.$$.fragment),Ls=t(),El=r("ul"),El.innerHTML=Ca,Ds=t(),M(Ql.$$.fragment),Ks=t(),Rl=r("p"),this.h()},l(l){const s=Aa("svelte-u9bgzb",document.head);c=j(s,"META",{name:!0,content:!0}),s.forEach(a),kl=e(l),Wl=j(l,"P",{}),ba(Wl).forEach(a),$l=e(l),p(u.$$.fragment,l),Sl=e(l),p(I.$$.fragment,l),vl=e(l),d=j(l,"P",{"data-svelte-h":!0}),o(d)!=="svelte-tu0igi"&&(d.textContent=sa),Vl=e(l),m=j(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(m)!=="svelte-18ku02z"&&(m.innerHTML=aa),_l=e(l),p(h.$$.fragment,l),Xl=e(l),C=j(l,"UL",{"data-svelte-h":!0}),o(C)!=="svelte-15itk3r"&&(C.innerHTML=na),Nl=e(l),p(b.$$.fragment,l),Fl=e(l),g=j(l,"P",{"data-svelte-h":!0}),o(g)!=="svelte-1dpktxz"&&(g.innerHTML=ta),Yl=e(l),B=j(l,"UL",{"data-svelte-h":!0}),o(B)!=="svelte-n9kbo0"&&(B.innerHTML=ea),zl=e(l),f=j(l,"P",{"data-svelte-h":!0}),o(f)!=="svelte-8dxgej"&&(f.textContent=Ma),ql=e(l),Z=j(l,"UL",{"data-svelte-h":!0}),o(Z)!=="svelte-1xz821d"&&(Z.innerHTML=pa),Hl=e(l),p(G.$$.fragment,l),xl=e(l),p(A.$$.fragment,l),Ll=e(l),p(E.$$.fragment,l),Dl=e(l),p(Q.$$.fragment,l),Kl=e(l),p(W.$$.fragment,l),Pl=e(l),p(R.$$.fragment,l),Ol=e(l),k=j(l,"P",{"data-svelte-h":!0}),o(k)!=="svelte-1xm4bsy"&&(k.textContent=ya),ls=e(l),p($.$$.fragment,l),ss=e(l),p(S.$$.fragment,l),as=e(l),p(v.$$.fragment,l),ns=e(l),p(V.$$.fragment,l),ts=e(l),p(_.$$.fragment,l),es=e(l),p(X.$$.fragment,l),Ms=e(l),N=j(l,"P",{"data-svelte-h":!0}),o(N)!=="svelte-zq8pmv"&&(N.textContent=ia),ps=e(l),F=j(l,"P",{"data-svelte-h":!0}),o(F)!=="svelte-1nta3bs"&&(F.innerHTML=Ja),ys=e(l),Y=j(l,"UL",{"data-svelte-h":!0}),o(Y)!=="svelte-1qt33pe"&&(Y.innerHTML=Ua),is=e(l),z=j(l,"P",{"data-svelte-h":!0}),o(z)!=="svelte-18ykmm8"&&(z.innerHTML=Ta),Js=e(l),p(q.$$.fragment,l),Us=e(l),p(H.$$.fragment,l),Ts=e(l),p(x.$$.fragment,l),rs=e(l),p(L.$$.fragment,l),js=e(l),p(D.$$.fragment,l),os=e(l),K=j(l,"P",{"data-svelte-h":!0}),o(K)!=="svelte-p925wf"&&(K.textContent=ra),ws=e(l),P=j(l,"OL",{"data-svelte-h":!0}),o(P)!=="svelte-m2sbcp"&&(P.innerHTML=ja),cs=e(l),p(O.$$.fragment,l),ms=e(l),p(ll.$$.fragment,l),us=e(l),p(sl.$$.fragment,l),Is=e(l),al=j(l,"P",{"data-svelte-h":!0}),o(al)!=="svelte-hpwesz"&&(al.textContent=oa),ds=e(l),p(nl.$$.fragment,l),hs=e(l),p(tl.$$.fragment,l),Cs=e(l),p(el.$$.fragment,l),bs=e(l),p(Ml.$$.fragment,l),gs=e(l),p(pl.$$.fragment,l),Bs=e(l),yl=j(l,"P",{"data-svelte-h":!0}),o(yl)!=="svelte-52iyj9"&&(yl.textContent=wa),fs=e(l),p(il.$$.fragment,l),Zs=e(l),p(Jl.$$.fragment,l),Gs=e(l),p(Ul.$$.fragment,l),As=e(l),p(Tl.$$.fragment,l),Es=e(l),p(rl.$$.fragment,l),Qs=e(l),jl=j(l,"P",{"data-svelte-h":!0}),o(jl)!=="svelte-nuvwd7"&&(jl.textContent=ca),Ws=e(l),p(ol.$$.fragment,l),Rs=e(l),p(wl.$$.fragment,l),ks=e(l),p(cl.$$.fragment,l),$s=e(l),p(ml.$$.fragment,l),Ss=e(l),p(ul.$$.fragment,l),vs=e(l),p(Il.$$.fragment,l),Vs=e(l),p(dl.$$.fragment,l),_s=e(l),hl=j(l,"P",{"data-svelte-h":!0}),o(hl)!=="svelte-1goylmh"&&(hl.textContent=ma),Xs=e(l),Cl=j(l,"OL",{"data-svelte-h":!0}),o(Cl)!=="svelte-tew79g"&&(Cl.innerHTML=ua),Ns=e(l),p(bl.$$.fragment,l),Fs=e(l),p(gl.$$.fragment,l),Ys=e(l),Bl=j(l,"P",{"data-svelte-h":!0}),o(Bl)!=="svelte-19gohz9"&&(Bl.textContent=Ia),zs=e(l),fl=j(l,"OL",{"data-svelte-h":!0}),o(fl)!=="svelte-ogol81"&&(fl.innerHTML=da),qs=e(l),p(Zl.$$.fragment,l),Hs=e(l),Gl=j(l,"UL",{"data-svelte-h":!0}),o(Gl)!=="svelte-jr79za"&&(Gl.innerHTML=ha),xs=e(l),p(Al.$$.fragment,l),Ls=e(l),El=j(l,"UL",{"data-svelte-h":!0}),o(El)!=="svelte-78nb3v"&&(El.innerHTML=Ca),Ds=e(l),p(Ql.$$.fragment,l),Ks=e(l),Rl=j(l,"P",{}),ba(Rl).forEach(a),this.h()},h(){Os(c,"name","hf:doc:metadata"),Os(c,"content",ka),Os(m,"class","note")},m(l,s){Ea(document.head,c),n(l,kl,s),n(l,Wl,s),n(l,$l,s),y(u,l,s),n(l,Sl,s),y(I,l,s),n(l,vl,s),n(l,d,s),n(l,Vl,s),n(l,m,s),n(l,_l,s),y(h,l,s),n(l,Xl,s),n(l,C,s),n(l,Nl,s),y(b,l,s),n(l,Fl,s),n(l,g,s),n(l,Yl,s),n(l,B,s),n(l,zl,s),n(l,f,s),n(l,ql,s),n(l,Z,s),n(l,Hl,s),y(G,l,s),n(l,xl,s),y(A,l,s),n(l,Ll,s),y(E,l,s),n(l,Dl,s),y(Q,l,s),n(l,Kl,s),y(W,l,s),n(l,Pl,s),y(R,l,s),n(l,Ol,s),n(l,k,s),n(l,ls,s),y($,l,s),n(l,ss,s),y(S,l,s),n(l,as,s),y(v,l,s),n(l,ns,s),y(V,l,s),n(l,ts,s),y(_,l,s),n(l,es,s),y(X,l,s),n(l,Ms,s),n(l,N,s),n(l,ps,s),n(l,F,s),n(l,ys,s),n(l,Y,s),n(l,is,s),n(l,z,s),n(l,Js,s),y(q,l,s),n(l,Us,s),y(H,l,s),n(l,Ts,s),y(x,l,s),n(l,rs,s),y(L,l,s),n(l,js,s),y(D,l,s),n(l,os,s),n(l,K,s),n(l,ws,s),n(l,P,s),n(l,cs,s),y(O,l,s),n(l,ms,s),y(ll,l,s),n(l,us,s),y(sl,l,s),n(l,Is,s),n(l,al,s),n(l,ds,s),y(nl,l,s),n(l,hs,s),y(tl,l,s),n(l,Cs,s),y(el,l,s),n(l,bs,s),y(Ml,l,s),n(l,gs,s),y(pl,l,s),n(l,Bs,s),n(l,yl,s),n(l,fs,s),y(il,l,s),n(l,Zs,s),y(Jl,l,s),n(l,Gs,s),y(Ul,l,s),n(l,As,s),y(Tl,l,s),n(l,Es,s),y(rl,l,s),n(l,Qs,s),n(l,jl,s),n(l,Ws,s),y(ol,l,s),n(l,Rs,s),y(wl,l,s),n(l,ks,s),y(cl,l,s),n(l,$s,s),y(ml,l,s),n(l,Ss,s),y(ul,l,s),n(l,vs,s),y(Il,l,s),n(l,Vs,s),y(dl,l,s),n(l,_s,s),n(l,hl,s),n(l,Xs,s),n(l,Cl,s),n(l,Ns,s),y(bl,l,s),n(l,Fs,s),y(gl,l,s),n(l,Ys,s),n(l,Bl,s),n(l,zs,s),n(l,fl,s),n(l,qs,s),y(Zl,l,s),n(l,Hs,s),n(l,Gl,s),n(l,xs,s),y(Al,l,s),n(l,Ls,s),n(l,El,s),n(l,Ds,s),y(Ql,l,s),n(l,Ks,s),n(l,Rl,s),Ps=!0},p:Ba,i(l){Ps||(i(u.$$.fragment,l),i(I.$$.fragment,l),i(h.$$.fragment,l),i(b.$$.fragment,l),i(G.$$.fragment,l),i(A.$$.fragment,l),i(E.$$.fragment,l),i(Q.$$.fragment,l),i(W.$$.fragment,l),i(R.$$.fragment,l),i($.$$.fragment,l),i(S.$$.fragment,l),i(v.$$.fragment,l),i(V.$$.fragment,l),i(_.$$.fragment,l),i(X.$$.fragment,l),i(q.$$.fragment,l),i(H.$$.fragment,l),i(x.$$.fragment,l),i(L.$$.fragment,l),i(D.$$.fragment,l),i(O.$$.fragment,l),i(ll.$$.fragment,l),i(sl.$$.fragment,l),i(nl.$$.fragment,l),i(tl.$$.fragment,l),i(el.$$.fragment,l),i(Ml.$$.fragment,l),i(pl.$$.fragment,l),i(il.$$.fragment,l),i(Jl.$$.fragment,l),i(Ul.$$.fragment,l),i(Tl.$$.fragment,l),i(rl.$$.fragment,l),i(ol.$$.fragment,l),i(wl.$$.fragment,l),i(cl.$$.fragment,l),i(ml.$$.fragment,l),i(ul.$$.fragment,l),i(Il.$$.fragment,l),i(dl.$$.fragment,l),i(bl.$$.fragment,l),i(gl.$$.fragment,l),i(Zl.$$.fragment,l),i(Al.$$.fragment,l),i(Ql.$$.fragment,l),Ps=!0)},o(l){J(u.$$.fragment,l),J(I.$$.fragment,l),J(h.$$.fragment,l),J(b.$$.fragment,l),J(G.$$.fragment,l),J(A.$$.fragment,l),J(E.$$.fragment,l),J(Q.$$.fragment,l),J(W.$$.fragment,l),J(R.$$.fragment,l),J($.$$.fragment,l),J(S.$$.fragment,l),J(v.$$.fragment,l),J(V.$$.fragment,l),J(_.$$.fragment,l),J(X.$$.fragment,l),J(q.$$.fragment,l),J(H.$$.fragment,l),J(x.$$.fragment,l),J(L.$$.fragment,l),J(D.$$.fragment,l),J(O.$$.fragment,l),J(ll.$$.fragment,l),J(sl.$$.fragment,l),J(nl.$$.fragment,l),J(tl.$$.fragment,l),J(el.$$.fragment,l),J(Ml.$$.fragment,l),J(pl.$$.fragment,l),J(il.$$.fragment,l),J(Jl.$$.fragment,l),J(Ul.$$.fragment,l),J(Tl.$$.fragment,l),J(rl.$$.fragment,l),J(ol.$$.fragment,l),J(wl.$$.fragment,l),J(cl.$$.fragment,l),J(ml.$$.fragment,l),J(ul.$$.fragment,l),J(Il.$$.fragment,l),J(dl.$$.fragment,l),J(bl.$$.fragment,l),J(gl.$$.fragment,l),J(Zl.$$.fragment,l),J(Al.$$.fragment,l),J(Ql.$$.fragment,l),Ps=!1},d(l){l&&(a(kl),a(Wl),a($l),a(Sl),a(vl),a(d),a(Vl),a(m),a(_l),a(Xl),a(C),a(Nl),a(Fl),a(g),a(Yl),a(B),a(zl),a(f),a(ql),a(Z),a(Hl),a(xl),a(Ll),a(Dl),a(Kl),a(Pl),a(Ol),a(k),a(ls),a(ss),a(as),a(ns),a(ts),a(es),a(Ms),a(N),a(ps),a(F),a(ys),a(Y),a(is),a(z),a(Js),a(Us),a(Ts),a(rs),a(js),a(os),a(K),a(ws),a(P),a(cs),a(ms),a(us),a(Is),a(al),a(ds),a(hs),a(Cs),a(bs),a(gs),a(Bs),a(yl),a(fs),a(Zs),a(Gs),a(As),a(Es),a(Qs),a(jl),a(Ws),a(Rs),a(ks),a($s),a(Ss),a(vs),a(Vs),a(_s),a(hl),a(Xs),a(Cl),a(Ns),a(Fs),a(Ys),a(Bl),a(zs),a(fl),a(qs),a(Hs),a(Gl),a(xs),a(Ls),a(El),a(Ds),a(Ks),a(Rl)),a(c),U(u,l),U(I,l),U(h,l),U(b,l),U(G,l),U(A,l),U(E,l),U(Q,l),U(W,l),U(R,l),U($,l),U(S,l),U(v,l),U(V,l),U(_,l),U(X,l),U(q,l),U(H,l),U(x,l),U(L,l),U(D,l),U(O,l),U(ll,l),U(sl,l),U(nl,l),U(tl,l),U(el,l),U(Ml,l),U(pl,l),U(il,l),U(Jl,l),U(Ul,l),U(Tl,l),U(rl,l),U(ol,l),U(wl,l),U(cl,l),U(ml,l),U(ul,l),U(Il,l),U(dl,l),U(bl,l),U(gl,l),U(Zl,l),U(Al,l),U(Ql,l)}}}const ka='{"title":"RL Training with OpenEnv: 2048 Game","local":"rl-training-with-openenv-2048-game","sections":[{"title":"What You’ll Learn","local":"what-youll-learn","sections":[],"depth":2},{"title":"Prerequisites","local":"prerequisites","sections":[],"depth":2},{"title":"Part 1: Environment Setup","local":"part-1-environment-setup","sections":[{"title":"Installation","local":"installation","sections":[],"depth":3},{"title":"Imports","local":"imports","sections":[],"depth":3}],"depth":2},{"title":"Part 2: Model Configuration","local":"part-2-model-configuration","sections":[{"title":"Configuration Classes","local":"configuration-classes","sections":[],"depth":3},{"title":"Loading the Model","local":"loading-the-model","sections":[],"depth":3}],"depth":2},{"title":"Part 3: The 2048 Environment","local":"part-3-the-2048-environment","sections":[{"title":"Game Overview","local":"game-overview","sections":[],"depth":3},{"title":"Connecting to the Environment","local":"connecting-to-the-environment","sections":[],"depth":3},{"title":"Board Utilities","local":"board-utilities","sections":[],"depth":3}],"depth":2},{"title":"Part 4: Reward Function Design","local":"part-4-reward-function-design","sections":[{"title":"Reward Implementation","local":"reward-implementation","sections":[],"depth":3}],"depth":2},{"title":"Part 5: Strategy Generation","local":"part-5-strategy-generation","sections":[{"title":"Prompt Template","local":"prompt-template","sections":[],"depth":3},{"title":"Executing Generated Strategies","local":"executing-generated-strategies","sections":[],"depth":3}],"depth":2},{"title":"Part 6: GRPO Training","local":"part-6-grpo-training","sections":[{"title":"Training Configuration","local":"training-configuration","sections":[],"depth":3},{"title":"Training Loop","local":"training-loop","sections":[],"depth":3}],"depth":2},{"title":"Part 7: Deployment","local":"part-7-deployment","sections":[{"title":"Saving the Model","local":"saving-the-model","sections":[],"depth":3},{"title":"Push to Hugging Face Hub","local":"push-to-hugging-face-hub","sections":[],"depth":3},{"title":"Using the Trained Model","local":"using-the-trained-model","sections":[],"depth":3}],"depth":2},{"title":"Preventing Reward Hacking","local":"preventing-reward-hacking","sections":[],"depth":2},{"title":"Summary","local":"summary","sections":[],"depth":2},{"title":"Next Steps","local":"next-steps","sections":[],"depth":2},{"title":"Related Resources","local":"related-resources","sections":[],"depth":2}],"depth":1}';function $a(la){return fa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Xa extends Za{constructor(c){super(),Ga(this,c,$a,Ra,ga,{})}}export{Xa as component};