openenv/pr_752/en/_app/immutable/nodes/49.3d91100a.js

import{s as Me,n as we,o as Te}from"../chunks/scheduler.2b22cead.js";import{S as ce,i as Je,e as i,s,c as p,h as fe,a as r,d as l,b as a,f as pe,g as m,j as R,k as ee,l as ye,m as n,n as M,t as w,o as T,p as c}from"../chunks/index.1a0e8013.js";import{C as de,H as L}from"../chunks/Heading.00bd83b1.js";import{C as me}from"../chunks/CodeBlock.89a9ab50.js";import{E as ue}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.b0d082a5.js";function je(te){let o,x,k,G,f,S,y,V,J,le="<p>This page is still being filled in. TRL integration is covered below; torchforge and SkyRL integrations are planned.</p>",W,d,ne="Use OpenEnv with popular RL frameworks like TRL, torchforge, and SkyRL.",Z,u,_,j,se="OpenEnv environments are designed to integrate seamlessly with RL training frameworks. The standard <code>step()</code>, <code>reset()</code>, <code>state()</code> API makes it easy to use environments in training loops.",X,g,Y,U,ae='<a href="https://huggingface.co/docs/trl" rel="nofollow">TRL (Transformer Reinforcement Learning)</a> is the recommended framework for training language models with RL.',Q,$,H,I,ie='See the <a href="../tutorials/wordle-grpo">Wordle with GRPO</a> tutorial for a complete example.',F,h,P,v,re="For custom training setups:",z,b,O,C,q,A,oe='<li><a href="rewards">Reward Design</a> - Design effective reward functions</li> <li><a href="../tutorials/wordle-grpo">Wordle with GRPO</a> - Complete TRL example</li>',N,B,K,E,D;return f=new de({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),y=new L({props:{title:"RL Framework Integration",local:"rl-framework-integration",headingTag:"h1"}}),u=new L({props:{title:"Overview",local:"overview",headingTag:"h2"}}),g=new L({props:{title:"TRL Integration",local:"trl-integration",headingTag:"h2"}}),$=new me({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTBBZnJvbSUyMG9wZW5lbnYlMjBpbXBvcnQlMjBBdXRvRW52JTJDJTIwQXV0b0FjdGlvbiUwQSUwQWVudiUyMCUzRCUyMEF1dG9FbnYuZnJvbV9lbnYoJTIydGV4dGFyZW5hJTIyKSUwQVRleHRBY3Rpb24lMjAlM0QlMjBBdXRvQWN0aW9uLmZyb21fZW52KCUyMnRleHRhcmVuYSUyMiklMEElMEElMjMlMjBVc2UlMjB3aXRoJTIwVFJMJ3MlMjBHUlBPJTIwdHJhaW5lciUwQXRyYWluZXIlMjAlM0QlMjBHUlBPVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwcmV3YXJkX21vZGVsJTNEcmV3YXJkX21vZGVsJTJDJTBBJTIwJTIwJTIwJTIwJTIzJTIwLi4uJTIwVFJMJTIwY29uZmlnJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer
<span class="hljs-keyword">from</span> openenv <span class="hljs-keyword">import</span> AutoEnv, AutoAction

env = AutoEnv.from_env(<span class="hljs-string">&quot;textarena&quot;</span>)
TextAction = AutoAction.from_env(<span class="hljs-string">&quot;textarena&quot;</span>)

<span class="hljs-comment"># Use with TRL&#x27;s GRPO trainer</span>
trainer = GRPOTrainer(
    model=model,
    reward_model=reward_model,
    <span class="hljs-comment"># ... TRL config</span>
)`,lang:"python",wrap:!1}}),h=new L({props:{title:"Generic Training Loop",local:"generic-training-loop",headingTag:"h2"}}),b=new me({props:{code:"ZnJvbSUyMG9wZW5lbnYlMjBpbXBvcnQlMjBBdXRvRW52JTJDJTIwQXV0b0FjdGlvbiUwQSUwQWVudiUyMCUzRCUyMEF1dG9FbnYuZnJvbV9lbnYoJTIybXktZW52JTIyKSUwQUFjdGlvbiUyMCUzRCUyMEF1dG9BY3Rpb24uZnJvbV9lbnYoJTIybXktZW52JTIyKSUwQSUwQXdpdGglMjBlbnYuc3luYygpJTIwYXMlMjBjbGllbnQlM0ElMEElMjAlMjAlMjAlMjBmb3IlMjBlcGlzb2RlJTIwaW4lMjByYW5nZShudW1fZXBpc29kZXMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwY2xpZW50LnJlc2V0KCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3aGlsZSUyMG5vdCUyMHJlc3VsdC50ZXJtaW5hdGVkJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwR2V0JTIwYWN0aW9uJTIwZnJvbSUyMHlvdXIlMjBwb2xpY3klMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhY3Rpb24lMjAlM0QlMjBwb2xpY3kocmVzdWx0Lm9ic2VydmF0aW9uKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFRha2UlMjBzdGVwJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwY2xpZW50LnN0ZXAoYWN0aW9uKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFVwZGF0ZSUyMHBvbGljeSUyMHdpdGglMjByZXdhcmQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwb2xpY3kudXBkYXRlKHJlc3VsdC5yZXdhcmQp",highlighted:`<span class="hljs-keyword">from</span> openenv <span class="hljs-keyword">import</span> AutoEnv, AutoAction

env = AutoEnv.from_env(<span class="hljs-string">&quot;my-env&quot;</span>)
Action = AutoAction.from_env(<span class="hljs-string">&quot;my-env&quot;</span>)

<span class="hljs-keyword">with</span> env.sync() <span class="hljs-keyword">as</span> client:
    <span class="hljs-keyword">for</span> episode <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_episodes):
        result = client.reset()

        <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> result.terminated:
            <span class="hljs-comment"># Get action from your policy</span>
            action = policy(result.observation)

            <span class="hljs-comment"># Take step</span>
            result = client.step(action)

            <span class="hljs-comment"># Update policy with reward</span>
            policy.update(result.reward)`,lang:"python",wrap:!1}}),C=new L({props:{title:"Next Steps",local:"next-steps",headingTag:"h2"}}),B=new ue({props:{source:"https://github.com/huggingface/openenv/blob/main/docs/source/guides/rl-integration.md"}}),{c(){o=i("meta"),x=s(),k=i("p"),G=s(),p(f.$$.fragment),S=s(),p(y.$$.fragment),V=s(),J=i("blockquote"),J.innerHTML=le,W=s(),d=i("p"),d.textContent=ne,Z=s(),p(u.$$.fragment),_=s(),j=i("p"),j.innerHTML=se,X=s(),p(g.$$.fragment),Y=s(),U=i("p"),U.innerHTML=ae,Q=s(),p($.$$.fragment),H=s(),I=i("p"),I.innerHTML=ie,F=s(),p(h.$$.fragment),P=s(),v=i("p"),v.textContent=re,z=s(),p(b.$$.fragment),O=s(),p(C.$$.fragment),q=s(),A=i("ul"),A.innerHTML=oe,N=s(),p(B.$$.fragment),K=s(),E=i("p"),this.h()},l(e){const t=fe("svelte-u9bgzb",document.head);o=r(t,"META",{name:!0,content:!0}),t.forEach(l),x=a(e),k=r(e,"P",{}),pe(k).forEach(l),G=a(e),m(f.$$.fragment,e),S=a(e),m(y.$$.fragment,e),V=a(e),J=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),R(J)!=="svelte-osv6gj"&&(J.innerHTML=le),W=a(e),d=r(e,"P",{"data-svelte-h":!0}),R(d)!=="svelte-ys5mim"&&(d.textContent=ne),Z=a(e),m(u.$$.fragment,e),_=a(e),j=r(e,"P",{"data-svelte-h":!0}),R(j)!=="svelte-1b57lzl"&&(j.innerHTML=se),X=a(e),m(g.$$.fragment,e),Y=a(e),U=r(e,"P",{"data-svelte-h":!0}),R(U)!=="svelte-y12km9"&&(U.innerHTML=ae),Q=a(e),m($.$$.fragment,e),H=a(e),I=r(e,"P",{"data-svelte-h":!0}),R(I)!=="svelte-15k97s6"&&(I.innerHTML=ie),F=a(e),m(h.$$.fragment,e),P=a(e),v=r(e,"P",{"data-svelte-h":!0}),R(v)!=="svelte-1i5buj4"&&(v.textContent=re),z=a(e),m(b.$$.fragment,e),O=a(e),m(C.$$.fragment,e),q=a(e),A=r(e,"UL",{"data-svelte-h":!0}),R(A)!=="svelte-186c4z9"&&(A.innerHTML=oe),N=a(e),m(B.$$.fragment,e),K=a(e),E=r(e,"P",{}),pe(E).forEach(l),this.h()},h(){ee(o,"name","hf:doc:metadata"),ee(o,"content",ge),ee(J,"class","note")},m(e,t){ye(document.head,o),n(e,x,t),n(e,k,t),n(e,G,t),M(f,e,t),n(e,S,t),M(y,e,t),n(e,V,t),n(e,J,t),n(e,W,t),n(e,d,t),n(e,Z,t),M(u,e,t),n(e,_,t),n(e,j,t),n(e,X,t),M(g,e,t),n(e,Y,t),n(e,U,t),n(e,Q,t),M($,e,t),n(e,H,t),n(e,I,t),n(e,F,t),M(h,e,t),n(e,P,t),n(e,v,t),n(e,z,t),M(b,e,t),n(e,O,t),M(C,e,t),n(e,q,t),n(e,A,t),n(e,N,t),M(B,e,t),n(e,K,t),n(e,E,t),D=!0},p:we,i(e){D||(w(f.$$.fragment,e),w(y.$$.fragment,e),w(u.$$.fragment,e),w(g.$$.fragment,e),w($.$$.fragment,e),w(h.$$.fragment,e),w(b.$$.fragment,e),w(C.$$.fragment,e),w(B.$$.fragment,e),D=!0)},o(e){T(f.$$.fragment,e),T(y.$$.fragment,e),T(u.$$.fragment,e),T(g.$$.fragment,e),T($.$$.fragment,e),T(h.$$.fragment,e),T(b.$$.fragment,e),T(C.$$.fragment,e),T(B.$$.fragment,e),D=!1},d(e){e&&(l(x),l(k),l(G),l(S),l(V),l(J),l(W),l(d),l(Z),l(_),l(j),l(X),l(Y),l(U),l(Q),l(H),l(I),l(F),l(P),l(v),l(z),l(O),l(q),l(A),l(N),l(K),l(E)),l(o),c(f,e),c(y,e),c(u,e),c(g,e),c($,e),c(h,e),c(b,e),c(C,e),c(B,e)}}}const ge='{"title":"RL Framework Integration","local":"rl-framework-integration","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"TRL Integration","local":"trl-integration","sections":[],"depth":2},{"title":"Generic Training Loop","local":"generic-training-loop","sections":[],"depth":2},{"title":"Next Steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';function Ue(te){return Te(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ce extends ce{constructor(o){super(),Je(this,o,Ue,je,Me,{})}}export{Ce as component};