openenv/pr_749/en/_app/immutable/nodes/48.8568aee3.js

import{s as De,n as Fe,o as Ke}from"../chunks/scheduler.2b22cead.js";import{S as Oe,i as qe,e as d,s as n,c as i,h as et,a as h,d as s,b as a,f as Xe,g as r,j as u,k as Qe,l as tt,m as l,n as p,t as o,o as m,p as c}from"../chunks/index.1a0e8013.js";import{C as st,H as f}from"../chunks/Heading.c0d3f116.js";import{C as K}from"../chunks/CodeBlock.c8d73295.js";import{E as lt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.21bcf336.js";function nt(_e){let M,O,D,q,g,ee,w,te,$,Re='Good reward signals are the single biggest lever on RL training outcomes — and the hardest thing to get right. This page covers the design principles that apply in OpenEnv and points to the <a href="../tutorials/rubrics">Rubrics tutorial</a> for the concrete implementation API.',se,y,le,T,Le="The OpenEnv contract is that reward computation stays on the server side, inside <code>Environment.step</code>. The <code>openenv.core.rubrics</code> module exists to make that computation composable — think <code>torch.nn.Module</code> but for reward functions. An environment declares <code>self.rubric</code> in its constructor and the base class helper <code>self._apply_rubric(action, observation)</code> runs it every step.",ne,j,We='See the <a href="../tutorials/rubrics">Rubrics tutorial</a> for the end-to-end walkthrough, including <code>WeightedSum</code>, <code>Gate</code>, <code>Sequential</code>, <code>LLMJudge</code>, and the trajectory rubrics used for delayed rewards like chess outcomes.',ae,J,ie,b,Ve="The principles below are independent of the rubric mechanics — they apply whether the reward is a single scalar or a composition of a dozen components.",re,v,pe,C,Be="Begin with a sparse success/failure signal. Only shape the reward after you’ve confirmed the agent can reach the positive signal at all.",oe,I,me,x,ce,U,Ge="Dense intermediate rewards speed up learning but invite reward hacking. Prefer adding shaping as a small-weighted <em>component</em> of a <code>WeightedSum</code>, so the dominant signal is still the real outcome.",de,A,he,k,ue,E,He="<li><strong>Sparse</strong> rewards (signal only on terminal transitions) are cleaner but slower to learn from.</li> <li><strong>Dense</strong> rewards (signal every step) are faster but can push the agent into local optima that optimise the proxy instead of the goal.</li>",fe,_,Se="For long-horizon tasks with a delayed outcome, reach for <code>TrajectoryRubric</code> and its built-in per-step credit assignment instead of hand-crafting a dense proxy.",Me,R,ge,L,we,W,$e,V,Ye="Rewards accumulate silently during the game, then the final step produces the outcome and <code>compute_step_rewards()</code> distributes it back with exponential discounting. This is exactly what <code>envs/chess_env/</code> ships with.",ye,B,Te,G,je,H,Ne="<code>Sequential</code> short-circuits to <code>0.0</code> the moment any child fails its gate, so expensive style / LLM judge calls never run on broken submissions.",Je,S,be,Y,ve,N,Pe="Per-turn progress helps the agent explore, but the terminal outcome still dominates the final score.",Ce,P,Ie,z,ze="<li><strong>Reward hacking.</strong> The agent optimises the proxy instead of the goal. Prefer shaping as a small-weighted component rather than a large intermediate bonus.</li> <li><strong>Sparse rewards that never fire.</strong> If the agent cannot reach the success signal in practice, training stalls. Measure success rate on a random policy before relying on a sparse reward.</li> <li><strong>Conflicting signals.</strong> Two criteria pulling in opposite directions produce a flat optimisation landscape. If you catch yourself subtracting rewards to “cancel out” a bad behaviour, consider a <code>Gate</code> instead — make the bad case a hard zero.</li> <li><strong>Component score drift.</strong> Without component-level logging, you won’t know which criterion dropped the total. Rubric introspection (<code>env.rubric.named_rubrics()</code> → <code>last_score</code>) gives you this for free.</li>",xe,Z,Ue,X,Ze='<li><a href="../tutorials/rubrics">Rubrics tutorial</a> — full API walkthrough with composable examples.</li> <li><a href="https://github.com/huggingface/OpenEnv/blob/main/rfcs/004-rubrics.md" rel="nofollow">RFC 004</a> — design rationale.</li> <li><a href="rl-integration">RL Framework Integration</a> — consume the reward signal in a training loop.</li> <li><a href="environment-anatomy">Environment Anatomy</a> — where the rubric plugs in.</li>',Ae,Q,ke,F,Ee;return g=new st({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new f({props:{title:"Reward Design",local:"reward-design",headingTag:"h1"}}),y=new f({props:{title:"Rewards Live Inside the Environment",local:"rewards-live-inside-the-environment",headingTag:"h2"}}),J=new f({props:{title:"Design Principles",local:"design-principles",headingTag:"h2"}}),v=new f({props:{title:"1. Start simple",local:"1-start-simple",headingTag:"h3"}}),I=new K({props:{code:"Y2xhc3MlMjBXaW5Mb3NzUnVicmljKFJ1YnJpYyklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBhY3Rpb24lMkMlMjBvYnNlcnZhdGlvbiklMjAtJTNFJTIwZmxvYXQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMG5vdCUyMG9ic2VydmF0aW9uLmRvbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAxLjAlMjBpZiUyMG9ic2VydmF0aW9uLnN1Y2Nlc3MlMjBlbHNlJTIwLTEuMA==",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">WinLossRubric</span>(<span class="hljs-title class_ inherited__">Rubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, action, observation</span>) -&gt; <span class="hljs-built_in">float</span>:
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> observation.done:
            <span class="hljs-keyword">return</span> <span class="hljs-number">0.0</span>
        <span class="hljs-keyword">return</span> <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> observation.success <span class="hljs-keyword">else</span> -<span class="hljs-number">1.0</span>`,lang:"python",wrap:!1}}),x=new f({props:{title:"2. Shape carefully",local:"2-shape-carefully",headingTag:"h3"}}),A=new K({props:{code:"cmV3YXJkJTIwJTNEJTIwV2VpZ2h0ZWRTdW0oJTBBJTIwJTIwJTIwJTIwJTVCV2luTG9zc1J1YnJpYygpJTJDJTIwUHJvZ3Jlc3NSdWJyaWMoKSU1RCUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodHMlM0QlNUIwLjglMkMlMjAwLjIlNUQlMkMlMEEp",highlighted:`reward = WeightedSum(
    [WinLossRubric(), ProgressRubric()],
    weights=[<span class="hljs-number">0.8</span>, <span class="hljs-number">0.2</span>],
)`,lang:"python",wrap:!1}}),k=new f({props:{title:"3. Consider density",local:"3-consider-density",headingTag:"h3"}}),R=new f({props:{title:"Worked Examples",local:"worked-examples",headingTag:"h2"}}),L=new f({props:{title:"Chess — sparse, trajectory-based",local:"chess--sparse-trajectory-based",headingTag:"h3"}}),W=new K({props:{code:"Y2xhc3MlMjBDaGVzc091dGNvbWVSdWJyaWMoRXhwb25lbnRpYWxEaXNjb3VudGluZ1RyYWplY3RvcnlSdWJyaWMpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwc2NvcmVfdHJhamVjdG9yeShzZWxmJTJDJTIwdHJhamVjdG9yeSklMjAtJTNFJTIwZmxvYXQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBfJTJDJTIwZmluYWxfb2JzJTIwJTNEJTIwdHJhamVjdG9yeSU1Qi0xJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwZmluYWxfb2JzLnJld2FyZCUyMCUyMCUyMCUyMyUyMCUyQjElMjAlMkYlMjAwJTIwJTJGJTIwLTElMjBmcm9tJTIwdGhlJTIwZW5naW5l",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">ChessOutcomeRubric</span>(<span class="hljs-title class_ inherited__">ExponentialDiscountingTrajectoryRubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">score_trajectory</span>(<span class="hljs-params">self, trajectory</span>) -&gt; <span class="hljs-built_in">float</span>:
        _, final_obs = trajectory[-<span class="hljs-number">1</span>]
        <span class="hljs-keyword">return</span> final_obs.reward   <span class="hljs-comment"># +1 / 0 / -1 from the engine</span>`,lang:"python",wrap:!1}}),B=new f({props:{title:"Coding — gated composition",local:"coding--gated-composition",headingTag:"h3"}}),G=new K({props:{code:"cmV3YXJkJTIwJTNEJTIwU2VxdWVudGlhbCglMEElMjAlMjAlMjAlMjBHYXRlKENvbXBpbGVzUnVicmljKCklMkMlMjB0aHJlc2hvbGQlM0QxLjApJTJDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwbXVzdCUyMGNvbXBpbGUlMEElMjAlMjAlMjAlMjBHYXRlKFRlc3RzUGFzc1J1YnJpYygpJTJDJTIwdGhyZXNob2xkJTNEMC41KSUyQyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMG11c3QlMjBwYXNzJTIwYXQlMjBsZWFzdCUyMGhhbGYlMjB0aGUlMjB0ZXN0cyUwQSUyMCUyMCUyMCUyMFdlaWdodGVkU3VtKCU1QlRlc3RzUGFzc1J1YnJpYygpJTJDJTIwU3R5bGVSdWJyaWMoKSU1RCUyQyUyMCU1QjAuNyUyQyUyMDAuMyU1RCklMkMlMEEp",highlighted:`reward = Sequential(
    Gate(CompilesRubric(), threshold=<span class="hljs-number">1.0</span>),              <span class="hljs-comment"># must compile</span>
    Gate(TestsPassRubric(), threshold=<span class="hljs-number">0.5</span>),             <span class="hljs-comment"># must pass at least half the tests</span>
    WeightedSum([TestsPassRubric(), StyleRubric()], [<span class="hljs-number">0.7</span>, <span class="hljs-number">0.3</span>]),
)`,lang:"python",wrap:!1}}),S=new f({props:{title:"Text tasks — mixed signal",local:"text-tasks--mixed-signal",headingTag:"h3"}}),Y=new K({props:{code:"cmV3YXJkJTIwJTNEJTIwV2VpZ2h0ZWRTdW0oJTBBJTIwJTIwJTIwJTIwJTVCV2luTG9zc1J1YnJpYygpJTJDJTIwUGVyVHVyblByb2dyZXNzUnVicmljKCklNUQlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRzJTNEJTVCMC43JTJDJTIwMC4zJTVEJTJDJTBBKQ==",highlighted:`reward = WeightedSum(
    [WinLossRubric(), PerTurnProgressRubric()],
    weights=[<span class="hljs-number">0.7</span>, <span class="hljs-number">0.3</span>],
)`,lang:"python",wrap:!1}}),P=new f({props:{title:"Common Pitfalls",local:"common-pitfalls",headingTag:"h2"}}),Z=new f({props:{title:"Next Steps",local:"next-steps",headingTag:"h2"}}),Q=new lt({props:{source:"https://github.com/huggingface/openenv/blob/main/docs/source/guides/rewards.md"}}),{c(){M=d("meta"),O=n(),D=d("p"),q=n(),i(g.$$.fragment),ee=n(),i(w.$$.fragment),te=n(),$=d("p"),$.innerHTML=Re,se=n(),i(y.$$.fragment),le=n(),T=d("p"),T.innerHTML=Le,ne=n(),j=d("p"),j.innerHTML=We,ae=n(),i(J.$$.fragment),ie=n(),b=d("p"),b.textContent=Ve,re=n(),i(v.$$.fragment),pe=n(),C=d("p"),C.textContent=Be,oe=n(),i(I.$$.fragment),me=n(),i(x.$$.fragment),ce=n(),U=d("p"),U.innerHTML=Ge,de=n(),i(A.$$.fragment),he=n(),i(k.$$.fragment),ue=n(),E=d("ul"),E.innerHTML=He,fe=n(),_=d("p"),_.innerHTML=Se,Me=n(),i(R.$$.fragment),ge=n(),i(L.$$.fragment),we=n(),i(W.$$.fragment),$e=n(),V=d("p"),V.innerHTML=Ye,ye=n(),i(B.$$.fragment),Te=n(),i(G.$$.fragment),je=n(),H=d("p"),H.innerHTML=Ne,Je=n(),i(S.$$.fragment),be=n(),i(Y.$$.fragment),ve=n(),N=d("p"),N.textContent=Pe,Ce=n(),i(P.$$.fragment),Ie=n(),z=d("ol"),z.innerHTML=ze,xe=n(),i(Z.$$.fragment),Ue=n(),X=d("ul"),X.innerHTML=Ze,Ae=n(),i(Q.$$.fragment),ke=n(),F=d("p"),this.h()},l(e){const t=et("svelte-u9bgzb",document.head);M=h(t,"META",{name:!0,content:!0}),t.forEach(s),O=a(e),D=h(e,"P",{}),Xe(D).forEach(s),q=a(e),r(g.$$.fragment,e),ee=a(e),r(w.$$.fragment,e),te=a(e),$=h(e,"P",{"data-svelte-h":!0}),u($)!=="svelte-1ah7f14"&&($.innerHTML=Re),se=a(e),r(y.$$.fragment,e),le=a(e),T=h(e,"P",{"data-svelte-h":!0}),u(T)!=="svelte-1016j2m"&&(T.innerHTML=Le),ne=a(e),j=h(e,"P",{"data-svelte-h":!0}),u(j)!=="svelte-17f70nv"&&(j.innerHTML=We),ae=a(e),r(J.$$.fragment,e),ie=a(e),b=h(e,"P",{"data-svelte-h":!0}),u(b)!=="svelte-l61ugv"&&(b.textContent=Ve),re=a(e),r(v.$$.fragment,e),pe=a(e),C=h(e,"P",{"data-svelte-h":!0}),u(C)!=="svelte-h3hcvc"&&(C.textContent=Be),oe=a(e),r(I.$$.fragment,e),me=a(e),r(x.$$.fragment,e),ce=a(e),U=h(e,"P",{"data-svelte-h":!0}),u(U)!=="svelte-722uui"&&(U.innerHTML=Ge),de=a(e),r(A.$$.fragment,e),he=a(e),r(k.$$.fragment,e),ue=a(e),E=h(e,"UL",{"data-svelte-h":!0}),u(E)!=="svelte-gl8mdx"&&(E.innerHTML=He),fe=a(e),_=h(e,"P",{"data-svelte-h":!0}),u(_)!=="svelte-m37xsb"&&(_.innerHTML=Se),Me=a(e),r(R.$$.fragment,e),ge=a(e),r(L.$$.fragment,e),we=a(e),r(W.$$.fragment,e),$e=a(e),V=h(e,"P",{"data-svelte-h":!0}),u(V)!=="svelte-c2nbf4"&&(V.innerHTML=Ye),ye=a(e),r(B.$$.fragment,e),Te=a(e),r(G.$$.fragment,e),je=a(e),H=h(e,"P",{"data-svelte-h":!0}),u(H)!=="svelte-18o41y"&&(H.innerHTML=Ne),Je=a(e),r(S.$$.fragment,e),be=a(e),r(Y.$$.fragment,e),ve=a(e),N=h(e,"P",{"data-svelte-h":!0}),u(N)!=="svelte-vybdqy"&&(N.textContent=Pe),Ce=a(e),r(P.$$.fragment,e),Ie=a(e),z=h(e,"OL",{"data-svelte-h":!0}),u(z)!=="svelte-18z52t9"&&(z.innerHTML=ze),xe=a(e),r(Z.$$.fragment,e),Ue=a(e),X=h(e,"UL",{"data-svelte-h":!0}),u(X)!=="svelte-a1i4y7"&&(X.innerHTML=Ze),Ae=a(e),r(Q.$$.fragment,e),ke=a(e),F=h(e,"P",{}),Xe(F).forEach(s),this.h()},h(){Qe(M,"name","hf:doc:metadata"),Qe(M,"content",at)},m(e,t){tt(document.head,M),l(e,O,t),l(e,D,t),l(e,q,t),p(g,e,t),l(e,ee,t),p(w,e,t),l(e,te,t),l(e,$,t),l(e,se,t),p(y,e,t),l(e,le,t),l(e,T,t),l(e,ne,t),l(e,j,t),l(e,ae,t),p(J,e,t),l(e,ie,t),l(e,b,t),l(e,re,t),p(v,e,t),l(e,pe,t),l(e,C,t),l(e,oe,t),p(I,e,t),l(e,me,t),p(x,e,t),l(e,ce,t),l(e,U,t),l(e,de,t),p(A,e,t),l(e,he,t),p(k,e,t),l(e,ue,t),l(e,E,t),l(e,fe,t),l(e,_,t),l(e,Me,t),p(R,e,t),l(e,ge,t),p(L,e,t),l(e,we,t),p(W,e,t),l(e,$e,t),l(e,V,t),l(e,ye,t),p(B,e,t),l(e,Te,t),p(G,e,t),l(e,je,t),l(e,H,t),l(e,Je,t),p(S,e,t),l(e,be,t),p(Y,e,t),l(e,ve,t),l(e,N,t),l(e,Ce,t),p(P,e,t),l(e,Ie,t),l(e,z,t),l(e,xe,t),p(Z,e,t),l(e,Ue,t),l(e,X,t),l(e,Ae,t),p(Q,e,t),l(e,ke,t),l(e,F,t),Ee=!0},p:Fe,i(e){Ee||(o(g.$$.fragment,e),o(w.$$.fragment,e),o(y.$$.fragment,e),o(J.$$.fragment,e),o(v.$$.fragment,e),o(I.$$.fragment,e),o(x.$$.fragment,e),o(A.$$.fragment,e),o(k.$$.fragment,e),o(R.$$.fragment,e),o(L.$$.fragment,e),o(W.$$.fragment,e),o(B.$$.fragment,e),o(G.$$.fragment,e),o(S.$$.fragment,e),o(Y.$$.fragment,e),o(P.$$.fragment,e),o(Z.$$.fragment,e),o(Q.$$.fragment,e),Ee=!0)},o(e){m(g.$$.fragment,e),m(w.$$.fragment,e),m(y.$$.fragment,e),m(J.$$.fragment,e),m(v.$$.fragment,e),m(I.$$.fragment,e),m(x.$$.fragment,e),m(A.$$.fragment,e),m(k.$$.fragment,e),m(R.$$.fragment,e),m(L.$$.fragment,e),m(W.$$.fragment,e),m(B.$$.fragment,e),m(G.$$.fragment,e),m(S.$$.fragment,e),m(Y.$$.fragment,e),m(P.$$.fragment,e),m(Z.$$.fragment,e),m(Q.$$.fragment,e),Ee=!1},d(e){e&&(s(O),s(D),s(q),s(ee),s(te),s($),s(se),s(le),s(T),s(ne),s(j),s(ae),s(ie),s(b),s(re),s(pe),s(C),s(oe),s(me),s(ce),s(U),s(de),s(he),s(ue),s(E),s(fe),s(_),s(Me),s(ge),s(we),s($e),s(V),s(ye),s(Te),s(je),s(H),s(Je),s(be),s(ve),s(N),s(Ce),s(Ie),s(z),s(xe),s(Ue),s(X),s(Ae),s(ke),s(F)),s(M),c(g,e),c(w,e),c(y,e),c(J,e),c(v,e),c(I,e),c(x,e),c(A,e),c(k,e),c(R,e),c(L,e),c(W,e),c(B,e),c(G,e),c(S,e),c(Y,e),c(P,e),c(Z,e),c(Q,e)}}}const at='{"title":"Reward Design","local":"reward-design","sections":[{"title":"Rewards Live Inside the Environment","local":"rewards-live-inside-the-environment","sections":[],"depth":2},{"title":"Design Principles","local":"design-principles","sections":[{"title":"1. Start simple","local":"1-start-simple","sections":[],"depth":3},{"title":"2. Shape carefully","local":"2-shape-carefully","sections":[],"depth":3},{"title":"3. Consider density","local":"3-consider-density","sections":[],"depth":3}],"depth":2},{"title":"Worked Examples","local":"worked-examples","sections":[{"title":"Chess — sparse, trajectory-based","local":"chess--sparse-trajectory-based","sections":[],"depth":3},{"title":"Coding — gated composition","local":"coding--gated-composition","sections":[],"depth":3},{"title":"Text tasks — mixed signal","local":"text-tasks--mixed-signal","sections":[],"depth":3}],"depth":2},{"title":"Common Pitfalls","local":"common-pitfalls","sections":[],"depth":2},{"title":"Next Steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';function it(_e){return Ke(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class dt extends Oe{constructor(M){super(),qe(this,M,it,nt,De,{})}}export{dt as component};