openenv/pr_1049/en/_app/immutable/nodes/52.BeEx4m4L.js

import"../chunks/DsnmJJEf.js";import{i as k,h as E,C as R,H as s,a,E as _,s as x}from"../chunks/C07d6Hje.js";import{p as W,o as V,s as e,f as B,a as v,b as G,c as I,n as Y}from"../chunks/BSuxAqoA.js";const S='{"title":"Reward Design","local":"reward-design","sections":[{"title":"Rewards Live Inside the Environment","local":"rewards-live-inside-the-environment","sections":[],"depth":2},{"title":"Design Principles","local":"design-principles","sections":[{"title":"1. Start simple","local":"1-start-simple","sections":[],"depth":3},{"title":"2. Shape carefully","local":"2-shape-carefully","sections":[],"depth":3},{"title":"3. Consider density","local":"3-consider-density","sections":[],"depth":3}],"depth":2},{"title":"Worked Examples","local":"worked-examples","sections":[{"title":"Chess — sparse, trajectory-based","local":"chess--sparse-trajectory-based","sections":[],"depth":3},{"title":"Coding — gated composition","local":"coding--gated-composition","sections":[],"depth":3},{"title":"Text tasks — mixed signal","local":"text-tasks--mixed-signal","sections":[],"depth":3}],"depth":2},{"title":"Common Pitfalls","local":"common-pitfalls","sections":[],"depth":2},{"title":"Next Steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';var N=I('<meta name="hf:doc:metadata"/>'),Z=I('<p></p> <!> <!> <p>Good reward signals are the single biggest lever on RL training outcomes — and the hardest thing to get right. This page covers the design principles that apply in OpenEnv and points to the <a href="../tutorials/rubrics">Rubrics tutorial</a> for the concrete implementation API.</p> <!> <p>The OpenEnv contract is that reward computation stays on the server side, inside <code>Environment.step</code>. The <code>openenv.core.rubrics</code> module exists to make that computation composable — think <code>torch.nn.Module</code> but for reward functions. An environment declares <code>self.rubric</code> in its constructor and the base class helper <code>self._apply_rubric(action, observation)</code> runs it every step.</p> <p>See the <a href="../tutorials/rubrics">Rubrics tutorial</a> for the end-to-end walkthrough, including <code>WeightedSum</code>, <code>Gate</code>, <code>Sequential</code>, <code>LLMJudge</code>, and the trajectory rubrics used for delayed rewards like chess outcomes.</p> <!> <p>The principles below are independent of the rubric mechanics — they apply whether the reward is a single scalar or a composition of a dozen components.</p> <!> <p>Begin with a sparse success/failure signal. Only shape the reward after you’ve confirmed the agent can reach the positive signal at all.</p> <!> <!> <p>Dense intermediate rewards speed up learning but invite reward hacking. Prefer adding shaping as a small-weighted <em>component</em> of a <code>WeightedSum</code>, so the dominant signal is still the real outcome.</p> <!> <!> <ul><li><strong>Sparse</strong> rewards (signal only on terminal transitions) are cleaner but slower to learn from.</li> <li><strong>Dense</strong> rewards (signal every step) are faster but can push the agent into local optima that optimise the proxy instead of the goal.</li></ul> <p>For long-horizon tasks with a delayed outcome, reach for <code>TrajectoryRubric</code> and its built-in per-step credit assignment instead of hand-crafting a dense proxy.</p> <!> <!> <!> <p>Rewards accumulate silently during the game, then the final step produces the outcome and <code>compute_step_rewards()</code> distributes it back with exponential discounting. This is exactly what <code>envs/chess_env/</code> ships with.</p> <!> <!> <p><code>Sequential</code> short-circuits to <code>0.0</code> the moment any child fails its gate, so expensive style / LLM judge calls never run on broken submissions.</p> <!> <!> <p>Per-turn progress helps the agent explore, but the terminal outcome still dominates the final score.</p> <!> <ol><li><strong>Reward hacking.</strong> The agent optimises the proxy instead of the goal. Prefer shaping as a small-weighted component rather than a large intermediate bonus.</li> <li><strong>Sparse rewards that never fire.</strong> If the agent cannot reach the success signal in practice, training stalls. Measure success rate on a random policy before relying on a sparse reward.</li> <li><strong>Conflicting signals.</strong> Two criteria pulling in opposite directions produce a flat optimisation landscape. If you catch yourself subtracting rewards to “cancel out” a bad behaviour, consider a <code>Gate</code> instead — make the bad case a hard zero.</li> <li><strong>Component score drift.</strong> Without component-level logging, you won’t know which criterion dropped the total. Rubric introspection (<code>env.rubric.named_rubrics()</code> → <code>last_score</code>) gives you this for free.</li></ol> <!> <ul><li><a href="../tutorials/rubrics">Rubrics tutorial</a> — full API walkthrough with composable examples.</li> <li><a href="https://github.com/huggingface/OpenEnv/blob/main/rfcs/004-rubrics.md" rel="nofollow">RFC 004</a> — design rationale.</li> <li><a href="rl-integration">RL Framework Integration</a> — consume the reward signal in a training loop.</li> <li><a href="concepts">Concepts</a> — where the rubric plugs in.</li></ul> <!> <p></p>',1);function Q(U,A){W(A,!1),V(()=>{new URLSearchParams(window.location.search).get("fw")}),k();var l=Z();E("19jjdiv",b=>{var f=N();x(f,"content",S),v(b,f)});var t=e(B(l),2);R(t,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var n=e(t,2);s(n,{title:"Reward Design",local:"reward-design",headingTag:"h1"});var i=e(n,4);s(i,{title:"Rewards Live Inside the Environment",local:"rewards-live-inside-the-environment",headingTag:"h2"});var o=e(i,6);s(o,{title:"Design Principles",local:"design-principles",headingTag:"h2"});var r=e(o,4);s(r,{title:"1. Start simple",local:"1-start-simple",headingTag:"h3"});var c=e(r,4);a(c,{code:"Y2xhc3MlMjBXaW5Mb3NzUnVicmljKFJ1YnJpYyklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBhY3Rpb24lMkMlMjBvYnNlcnZhdGlvbiklMjAtJTNFJTIwZmxvYXQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMG5vdCUyMG9ic2VydmF0aW9uLmRvbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAxLjAlMjBpZiUyMG9ic2VydmF0aW9uLnN1Y2Nlc3MlMjBlbHNlJTIwLTEuMA==",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">WinLossRubric</span>(<span class="hljs-title class_ inherited__">Rubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, action, observation</span>) -&gt; <span class="hljs-built_in">float</span>:
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> observation.done:
            <span class="hljs-keyword">return</span> <span class="hljs-number">0.0</span>
        <span class="hljs-keyword">return</span> <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> observation.success <span class="hljs-keyword">else</span> -<span class="hljs-number">1.0</span>`,lang:"python",wrap:!1});var d=e(c,2);s(d,{title:"2. Shape carefully",local:"2-shape-carefully",headingTag:"h3"});var p=e(d,4);a(p,{code:"cmV3YXJkJTIwJTNEJTIwV2VpZ2h0ZWRTdW0oJTBBJTIwJTIwJTIwJTIwJTVCV2luTG9zc1J1YnJpYygpJTJDJTIwUHJvZ3Jlc3NSdWJyaWMoKSU1RCUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodHMlM0QlNUIwLjglMkMlMjAwLjIlNUQlMkMlMEEp",highlighted:`reward = WeightedSum(
    [WinLossRubric(), ProgressRubric()],
    weights=[<span class="hljs-number">0.8</span>, <span class="hljs-number">0.2</span>],
)`,lang:"python",wrap:!1});var h=e(p,2);s(h,{title:"3. Consider density",local:"3-consider-density",headingTag:"h3"});var g=e(h,6);s(g,{title:"Worked Examples",local:"worked-examples",headingTag:"h2"});var u=e(g,2);s(u,{title:"Chess — sparse, trajectory-based",local:"chess--sparse-trajectory-based",headingTag:"h3"});var m=e(u,2);a(m,{code:"Y2xhc3MlMjBDaGVzc091dGNvbWVSdWJyaWMoRXhwb25lbnRpYWxEaXNjb3VudGluZ1RyYWplY3RvcnlSdWJyaWMpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwc2NvcmVfdHJhamVjdG9yeShzZWxmJTJDJTIwdHJhamVjdG9yeSklMjAtJTNFJTIwZmxvYXQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBfJTJDJTIwZmluYWxfb2JzJTIwJTNEJTIwdHJhamVjdG9yeSU1Qi0xJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwZmluYWxfb2JzLnJld2FyZCUyMCUyMCUyMCUyMyUyMCUyQjElMjAlMkYlMjAwJTIwJTJGJTIwLTElMjBmcm9tJTIwdGhlJTIwZW5naW5l",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">ChessOutcomeRubric</span>(<span class="hljs-title class_ inherited__">ExponentialDiscountingTrajectoryRubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">score_trajectory</span>(<span class="hljs-params">self, trajectory</span>) -&gt; <span class="hljs-built_in">float</span>:
        _, final_obs = trajectory[-<span class="hljs-number">1</span>]
        <span class="hljs-keyword">return</span> final_obs.reward   <span class="hljs-comment"># +1 / 0 / -1 from the engine</span>`,lang:"python",wrap:!1});var M=e(m,4);s(M,{title:"Coding — gated composition",local:"coding--gated-composition",headingTag:"h3"});var w=e(M,2);a(w,{code:"cmV3YXJkJTIwJTNEJTIwU2VxdWVudGlhbCglMEElMjAlMjAlMjAlMjBHYXRlKENvbXBpbGVzUnVicmljKCklMkMlMjB0aHJlc2hvbGQlM0QxLjApJTJDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwbXVzdCUyMGNvbXBpbGUlMEElMjAlMjAlMjAlMjBHYXRlKFRlc3RzUGFzc1J1YnJpYygpJTJDJTIwdGhyZXNob2xkJTNEMC41KSUyQyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMG11c3QlMjBwYXNzJTIwYXQlMjBsZWFzdCUyMGhhbGYlMjB0aGUlMjB0ZXN0cyUwQSUyMCUyMCUyMCUyMFdlaWdodGVkU3VtKCU1QlRlc3RzUGFzc1J1YnJpYygpJTJDJTIwU3R5bGVSdWJyaWMoKSU1RCUyQyUyMCU1QjAuNyUyQyUyMDAuMyU1RCklMkMlMEEp",highlighted:`reward = Sequential(
    Gate(CompilesRubric(), threshold=<span class="hljs-number">1.0</span>),              <span class="hljs-comment"># must compile</span>
    Gate(TestsPassRubric(), threshold=<span class="hljs-number">0.5</span>),             <span class="hljs-comment"># must pass at least half the tests</span>
    WeightedSum([TestsPassRubric(), StyleRubric()], [<span class="hljs-number">0.7</span>, <span class="hljs-number">0.3</span>]),
)`,lang:"python",wrap:!1});var j=e(w,4);s(j,{title:"Text tasks — mixed signal",local:"text-tasks--mixed-signal",headingTag:"h3"});var y=e(j,2);a(y,{code:"cmV3YXJkJTIwJTNEJTIwV2VpZ2h0ZWRTdW0oJTBBJTIwJTIwJTIwJTIwJTVCV2luTG9zc1J1YnJpYygpJTJDJTIwUGVyVHVyblByb2dyZXNzUnVicmljKCklNUQlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRzJTNEJTVCMC43JTJDJTIwMC4zJTVEJTJDJTBBKQ==",highlighted:`reward = WeightedSum(
    [WinLossRubric(), PerTurnProgressRubric()],
    weights=[<span class="hljs-number">0.7</span>, <span class="hljs-number">0.3</span>],
)`,lang:"python",wrap:!1});var J=e(y,4);s(J,{title:"Common Pitfalls",local:"common-pitfalls",headingTag:"h2"});var T=e(J,4);s(T,{title:"Next Steps",local:"next-steps",headingTag:"h2"});var C=e(T,4);_(C,{source:"https://github.com/huggingface/openenv/blob/main/docs/source/guides/rewards.md"}),Y(2),v(U,l),G()}export{Q as component};