openenv/pr_764/en/_app/immutable/nodes/64.ca8d4f8a.js

import{s as rt,n as ot,o as ct}from"../chunks/scheduler.2b22cead.js";import{S as pt,i as Mt,e as i,s as n,c,h as dt,a as r,d as l,b as a,f as it,g as p,j as o,k as Xe,l as yt,m as t,n as M,t as d,o as y,p as j}from"../chunks/index.1a0e8013.js";import{C as jt,H as w}from"../chunks/Heading.0262ca46.js";import{C as u}from"../chunks/CodeBlock.98ef6f09.js";import{E as wt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e23afded.js";function ut(ul){let J,xe,Qe,ze,b,Le,U,He,f,Jl='<a href="https://colab.research.google.com/github/huggingface/OpenEnv/blob/main/examples/rubrics.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>',Fe,C,hl="Rubrics are OpenEnv’s first-class abstraction for computing rewards. They let you build multi-criteria reward functions from small reusable pieces. This tutorial walks through the API end-to-end, from a one-line rubric to a full environment that introspects its reward signal at training time.",De,I,qe,g,ml="Before rubrics, each environment rolled its own reward logic. Three pain points surfaced repeatedly:",Pe,v,Tl="<li><strong>No standard interface</strong>. Every environment author invented their own <code>compute_reward(...)</code> shape, so reusing a reward component across environments meant copy-pasting.</li> <li><strong>Multi-criteria evaluation was ad-hoc</strong>. “Code must compile, tests must pass, style matters a bit” becomes a tangle of nested <code>if</code>/<code>else</code> and hand-rolled weighted averages. There was no consistent way to ask <em>which</em> criterion caused a low reward.</li> <li><strong>LLM judges and sandboxed checks are slow</strong>. Without a framework-level concept of “reward component”, batch evaluation couldn’t parallelise the I/O-bound pieces.</li>",Ke,B,bl="The Rubric API is small: you subclass, implement <code>forward</code>, and the framework gives you composition, introspection, and parallel evaluation for free.",Oe,A,es,Z,Ul="A rubric is a callable with a <code>forward(action, observation) -&gt; float</code> method.",ss,_,ls,k,fl="That’s the whole contract. Instantiate it and call it:",ts,R,ns,W,Cl="<code>Rubric.__call__</code> runs pre- and post-hooks around your <code>forward</code>, caches the result on <code>self.last_score</code>, and supports async <code>forward</code> implementations transparently. (If you’ve used <code>torch.nn.Module</code>, the subclass-and-implement-<code>forward</code> pattern will feel familiar — children assigned as instance attributes auto-register with the parent.)",as,G,is,$,Il="You can attach hooks without subclassing — useful for logging every component’s score without polluting <code>forward</code>. Post-hooks run after <code>forward</code> completes and see the returned score; pre-hooks run before <code>forward</code> and are handy for input validation or instrumentation. When a rubric is async, hooks are awaited transparently.",rs,Y,os,E,cs,N,gl="Rubrics implement <code>state_dict()</code> / <code>load_state_dict(state)</code> so their configuration (thresholds, prompt templates, etc.) can be serialised alongside model checkpoints. The default implementations return an empty dict — override them when your rubric has tunable parameters.",ps,S,Ms,Q,vl="The real power shows up when you stack rubrics. <code>openenv.core.rubrics</code> ships with four containers.",ds,V,ys,X,Bl="Use when several independent criteria each contribute to the final score.",js,x,ws,z,Al="Weights must sum to <code>1.0</code>. <code>WeightedSum</code> evaluates its children with <code>asyncio.gather</code> when any of them is async, so an LLM-backed child does not block the synchronous ones.",us,L,Js,H,Zl="Use when a child score below a threshold should short-circuit the reward to zero.",hs,F,ms,D,_l="<code>Gate</code> returns <code>0.0</code> when the child score is below the threshold, and passes the child score through unchanged otherwise.",Ts,q,bs,P,kl="Use when criteria are ordered: a later criterion only matters if the earlier ones passed. Sequential returns <code>0.0</code> the moment any child returns <code>0.0</code> and does not evaluate the remaining children — great for gating expensive checks like sandboxed test runs or LLM calls.",Us,K,fs,O,Cs,ee,Rl="When the right rubric depends on the current observation (e.g. one rubric per game in a multi-game environment), wrap the options in a <code>RubricList</code> or <code>RubricDict</code> and dispatch in your parent rubric’s <code>forward</code>.",Is,se,gs,le,Wl="<code>RubricList</code> and <code>RubricDict</code> do not aggregate on their own — calling them directly raises. Their job is auto-registration (so their children show up in <code>named_rubrics()</code>) and indexed access. Reach for them when the parent rubric needs to pick a child <em>at runtime</em> based on the observation — if the set of children is fixed, plain attributes are simpler.",vs,te,Bs,ne,Gl="Assigning a child rubric as an attribute auto-registers it with the parent. Training code can then walk the tree:",As,ae,Zs,ie,$l="After running the composite once, every component’s most recent score is cached on <code>last_score</code> — no manual bookkeeping.",_s,re,ks,oe,Yl="When a criterion is too subjective for a handwritten heuristic (“is this argument persuasive?”, “is this explanation clear?”), use an LLM as the judge. <code>LLMJudge</code> wraps an <code>LLMClient</code> with a prompt template and a score extractor.",Rs,ce,El="Any OpenAI-compatible endpoint works: hosted OpenAI / Anthropic, or open-weight models served through vLLM, Ollama, Hugging Face Inference Providers, etc. Pick a client and hand it to <code>LLMJudge</code>:",Ws,pe,Gs,Me,Nl="<code>LLMJudge.forward</code> is async. When you put it inside <code>WeightedSum</code> or <code>Sequential</code>, the container awaits it transparently. A few caveats worth stating up front:",$s,de,Sl="<li><strong>Cost and latency</strong> scale with the number of episodes and the number of rubric calls per step. <code>Sequential</code> + <code>Gate</code> earlier in the pipeline is the usual answer.</li> <li><strong>Determinism</strong> is not free. Cache scores when you can, and consider temperature 0 for repeatable eval runs.</li> <li><strong>API keys</strong> belong in environment variables (<code>OPENAI_API_KEY</code>, <code>ANTHROPIC_API_KEY</code>, …), not in code that ships to the Hub.</li>",Ys,ye,Es,je,Ql="Some signals only materialise at the end of an episode — chess win/loss, unit-test suite success, a goal reached after many steps. <code>TrajectoryRubric</code> accumulates <code>(action, observation)</code> pairs internally and only invokes your scoring logic on the terminal observation.",Ns,we,Ss,ue,Vl="<code>forward(action, obs)</code> returns <code>intermediate_reward</code> (default <code>0.0</code>) until <code>observation.done</code> is <code>True</code>, then calls <code>score_trajectory</code>. After the episode ends, call <code>rubric.compute_step_rewards()</code> to get one reward per step — same length as the trajectory. This is the hook for credit assignment: training code feeds these per-step rewards back into advantage estimation, return-to-go, or whatever your optimizer expects. <code>ExponentialDiscountingTrajectoryRubric</code> precomputes <code>gamma^(T-1-t) * final_score</code> for you; override <code>compute_step_rewards</code> in your subclass if you want a different strategy (all-to-last, equal split, task-specific shaping).",Qs,h,Xl="<p>If <code>observation.done</code> never becomes <code>True</code>, <code>score_trajectory</code> is never called and the trajectory grows unbounded in memory. Make sure <code>step</code> flips <code>done</code> on every terminal transition, and call <code>self._reset_rubric()</code> in <code>Environment.reset</code> so trajectories do not leak across episodes.</p>",Vs,Je,xl="For the common exponentially-discounted case, subclass <code>ExponentialDiscountingTrajectoryRubric</code> instead and only implement <code>score_trajectory</code>:",Xs,he,xs,me,zl="This is exactly the pattern the built-in <code>envs/chess_env/</code> uses — see <code>envs/chess_env/server/rubrics.py</code> for the complete real-world example.",zs,m,Ll="<p>The <code>TrajectoryRubric</code> keeps the trajectory in CPU memory. If your observation carries GPU tensors (images, embeddings), detach and move them to CPU before returning from <code>step()</code> — otherwise the trajectory holds onto GPU memory across the whole episode.</p>",Ls,Te,Hs,be,Hl="Rubrics are <strong>server-side</strong>. Each environment declares its rubric in <code>__init__</code>, and <code>step</code> runs it via the <code>_apply_rubric</code> helper. The base <code>Environment</code> class accepts the rubric through its constructor and stores it as <code>self.rubric</code>.",Fs,Ue,Fl="Here is a complete minimal environment that composes a <code>Sequential</code> gate-then-<code>WeightedSum</code> pipeline and exposes the reward through its observation:",Ds,fe,qs,Ce,Dl="The three pieces the base class expects from you:",Ps,Ie,ql="<li><strong>Pass the rubric to <code>super().__init__(rubric=...)</code></strong> so <code>self.rubric</code> is set.</li> <li><strong>Call <code>self._reset_rubric()</code> from <code>reset</code></strong> so trajectory state does not leak between episodes.</li> <li><strong>Call <code>self._apply_rubric(action, obs)</code> from <code>step</code></strong> and attach the result to <code>obs.reward</code>. There is also <code>_apply_rubric_async</code> for <code>step_async</code>.</li>",Ks,T,Pl="<p>Some environments already compute <code>obs.reward</code> from game mechanics or a handcrafted multi-component signal (see <code>envs/chess_env/</code> and <code>envs/carla_env/</code>). In that case, call <code>self._apply_rubric(action, obs)</code> without assigning its return value — the rubric still accumulates the trajectory for <code>compute_step_rewards()</code> and still exposes per-component scores via <code>named_rubrics()</code>, but <code>obs.reward</code> stays authoritative.</p>",Os,ge,el,ve,Kl="Because children are auto-registered, the training loop can walk the rubric tree and log component-level diagnostics without the environment exposing a custom API:",sl,Be,ll,Ae,Ol="That snippet works for <em>any</em> OpenEnv environment that sets <code>self.rubric</code>, regardless of whether the rubric is a single scalar or a deeply nested composition.",tl,Ze,nl,_e,et="Training frameworks consume the reward through the same channel as any other OpenEnv observation field: <code>step()</code> returns an <code>Observation</code> whose <code>reward</code> is the rubric’s output, and the client delivers it via <code>result.reward</code>.",al,ke,st='With <a href="https://huggingface.co/docs/trl/main/en/openenv" rel="nofollow">TRL</a>, the recommended path is <code>GRPOTrainer</code>’s <code>environment_factory</code>: you define a thin wrapper class with tool methods that call the OpenEnv client, store <code>self.reward = result.observation.reward</code> after each step, and a plain reward function reads it off the <code>environments</code> parameter. The <a href="https://huggingface.co/docs/trl/main/en/openenv" rel="nofollow">TRL OpenEnv integration guide</a> has the full recipe, and <a href="https://github.com/huggingface/trl/tree/main/examples/scripts/openenv" rel="nofollow"><code>examples/scripts/openenv/</code></a> ships ready-to-run scripts. The same observation shape works with <a href="https://github.com/pytorch-labs/torchforge" rel="nofollow">torchforge</a> and other OpenEnv-compatible training stacks.',il,Re,lt="<code>named_rubrics()</code> is orthogonal: use it to <strong>log per-component scores</strong> (to Weights &amp; Biases, TensorBoard, trackio, …) while training, without changing the reward the optimiser sees.",rl,We,ol,Ge,tt="A rubric is just a callable — nothing forces you to run it inside a training loop. Drop it into a for-loop over a static dataset and you have a multi-criteria scoring function for offline eval:",cl,$e,pl,Ye,nt="The same rubric object used to compute training rewards doubles as the eval metric — one source of truth for “what is a good response”. Per-component <code>last_score</code> gives you a per-criterion breakdown for free (useful for regression dashboards and failure analysis). When a component like <code>LLMJudge</code> is async, wrap the loop with <code>asyncio.run(...)</code> and <code>await rubric(action, obs)</code> so the judge calls can overlap.",Ml,Ee,dl,Ne,at='<li><strong>Real-world trajectory example</strong> — walk through <code>envs/chess_env/server/rubrics.py</code> and <code>chess_environment.py</code> to see <code>ExponentialDiscountingTrajectoryRubric</code> wired into a game environment.</li> <li><strong>Design details</strong> — <a href="https://github.com/huggingface/OpenEnv/blob/main/rfcs/004-rubrics.md" rel="nofollow">RFC 004</a> covers the rationale for the composable API and the “rewards inside the environment” invariant.</li> <li><strong>Reward design basics</strong> — the <a href="../guides/rewards">Reward Design</a> guide covers sparse-vs-dense signals and common pitfalls that still apply on top of any rubric composition.</li> <li><strong>Training loop integration</strong> — see the <a href="../guides/rl-integration">RL Framework Integration</a> guide and the <a href="https://huggingface.co/docs/trl/main/en/openenv" rel="nofollow">TRL OpenEnv integration guide</a> for the recommended <code>environment_factory</code> pattern.</li>',yl,Se,jl,Ve,wl;return b=new jt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new w({props:{title:"Rubrics: Composable Reward Computation",local:"rubrics-composable-reward-computation",headingTag:"h1"}}),I=new w({props:{title:"Why Rubrics?",local:"why-rubrics",headingTag:"h2"}}),A=new w({props:{title:"Your First Rubric",local:"your-first-rubric",headingTag:"h2"}}),_=new u({props:{code:"ZnJvbSUyMG9wZW5lbnYuY29yZS5ydWJyaWNzJTIwaW1wb3J0JTIwUnVicmljJTBBJTBBJTBBY2xhc3MlMjBNZXNzYWdlTGVuZ3RoUnVicmljKFJ1YnJpYyklM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjJSZXdhcmQlMjAxLjAlMjBpZiUyMHRoZSUyMG1lc3NhZ2UlMjBpcyUyMDUlRTIlODAlOTMyMCUyMGNoYXJhY3RlcnMlMjBsb25nJTJDJTIwZWxzZSUyMDAuMC4lMjIlMjIlMjIlMEElMEElMjAlMjAlMjAlMjBkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBhY3Rpb24lMkMlMjBvYnNlcnZhdGlvbiklMjAtJTNFJTIwZmxvYXQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsZW5ndGglMjAlM0QlMjBsZW4oYWN0aW9uLm1lc3NhZ2UpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwMS4wJTIwaWYlMjA1JTIwJTNDJTNEJTIwbGVuZ3RoJTIwJTNDJTNEJTIwMjAlMjBlbHNlJTIwMC4w",highlighted:`<span class="hljs-keyword">from</span> openenv.core.rubrics <span class="hljs-keyword">import</span> Rubric


<span class="hljs-keyword">class</span> <span class="hljs-title class_">MessageLengthRubric</span>(<span class="hljs-title class_ inherited__">Rubric</span>):
    <span class="hljs-string">&quot;&quot;&quot;Reward 1.0 if the message is 5–20 characters long, else 0.0.&quot;&quot;&quot;</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, action, observation</span>) -&gt; <span class="hljs-built_in">float</span>:
        length = <span class="hljs-built_in">len</span>(action.message)
        <span class="hljs-keyword">return</span> <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> <span class="hljs-number">5</span> &lt;= length &lt;= <span class="hljs-number">20</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span>`,lang:"python",wrap:!1}}),R=new u({props:{code:"cnVicmljJTIwJTNEJTIwTWVzc2FnZUxlbmd0aFJ1YnJpYygpJTBBc2NvcmUlMjAlM0QlMjBydWJyaWMoYWN0aW9uJTJDJTIwb2JzZXJ2YXRpb24pJTIwJTIwJTIwJTIzJTIwcnVucyUyMGZvcndhcmQlMjAlMkIlMjBob29rcyUwQXByaW50KHJ1YnJpYy5sYXN0X3Njb3JlKSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGxhdGVzdCUyMHNjb3JlJTIwaXMlMjBjYWNoZWQlMjBvbiUyMHRoZSUyMHJ1YnJpYw==",highlighted:`rubric = MessageLengthRubric()
score = rubric(action, observation)   <span class="hljs-comment"># runs forward + hooks</span>
<span class="hljs-built_in">print</span>(rubric.last_score)              <span class="hljs-comment"># latest score is cached on the rubric</span>`,lang:"python",wrap:!1}}),G=new w({props:{title:"Optional hooks for observability",local:"optional-hooks-for-observability",headingTag:"h3"}}),Y=new u({props:{code:"ZGVmJTIwbG9nX3Njb3JlKHJ1YnJpYyUyQyUyMGFjdGlvbiUyQyUyMG9icyUyQyUyMHJlc3VsdCklM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyJTdCdHlwZShydWJyaWMpLl9fbmFtZV9fJTdEJTNBJTIwJTdCcmVzdWx0JTNBLjJmJTdEJTIyKSUwQSUwQXJ1YnJpYy5yZWdpc3Rlcl9mb3J3YXJkX2hvb2sobG9nX3Njb3JlKSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGZpcmVzJTIwYWZ0ZXIlMjBmb3J3YXJkKCklMEFydWJyaWMucmVnaXN0ZXJfZm9yd2FyZF9wcmVfaG9vayhsYW1iZGElMjByJTJDJTIwYSUyQyUyMG8lM0ElMjBOb25lKSUyMCUyMCUyMyUyMGZpcmVzJTIwYmVmb3JlJTIwZm9yd2FyZCgp",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">log_score</span>(<span class="hljs-params">rubric, action, obs, result</span>):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{<span class="hljs-built_in">type</span>(rubric).__name__}</span>: <span class="hljs-subst">{result:<span class="hljs-number">.2</span>f}</span>&quot;</span>)

rubric.register_forward_hook(log_score)              <span class="hljs-comment"># fires after forward()</span>
rubric.register_forward_pre_hook(<span class="hljs-keyword">lambda</span> r, a, o: <span class="hljs-literal">None</span>)  <span class="hljs-comment"># fires before forward()</span>`,lang:"python",wrap:!1}}),E=new w({props:{title:"State dict",local:"state-dict",headingTag:"h3"}}),S=new w({props:{title:"Composing Rubrics",local:"composing-rubrics",headingTag:"h2"}}),V=new w({props:{title:"WeightedSum — multi-criteria averaging",local:"weightedsum--multi-criteria-averaging",headingTag:"h3"}}),x=new u({props:{code:"ZnJvbSUyMG9wZW5lbnYuY29yZS5ydWJyaWNzJTIwaW1wb3J0JTIwV2VpZ2h0ZWRTdW0lMEElMEElMEFjbGFzcyUyMFRlc3RzUGFzc1J1YnJpYyhSdWJyaWMpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwZm9yd2FyZChzZWxmJTJDJTIwYWN0aW9uJTJDJTIwb2JzZXJ2YXRpb24pJTIwLSUzRSUyMGZsb2F0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwb2JzZXJ2YXRpb24udGVzdHNfcGFzc2VkJTIwJTJGJTIwbWF4KG9ic2VydmF0aW9uLnRlc3RzX3RvdGFsJTJDJTIwMSklMEElMEElMEFjbGFzcyUyMFN0eWxlUnVicmljKFJ1YnJpYyklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBhY3Rpb24lMkMlMjBvYnNlcnZhdGlvbiklMjAtJTNFJTIwZmxvYXQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAxLjAlMjBpZiUyMGFjdGlvbi5jb2RlLmNvdW50KCUyMiU1Q24lNUNuJTVDbiUyMiklMjAlM0QlM0QlMjAwJTIwZWxzZSUyMDAuNiUwQSUwQSUwQXJld2FyZCUyMCUzRCUyMFdlaWdodGVkU3VtKCUwQSUyMCUyMCUyMCUyMCU1QlRlc3RzUGFzc1J1YnJpYygpJTJDJTIwU3R5bGVSdWJyaWMoKSU1RCUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodHMlM0QlNUIwLjclMkMlMjAwLjMlNUQlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> openenv.core.rubrics <span class="hljs-keyword">import</span> WeightedSum


<span class="hljs-keyword">class</span> <span class="hljs-title class_">TestsPassRubric</span>(<span class="hljs-title class_ inherited__">Rubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, action, observation</span>) -&gt; <span class="hljs-built_in">float</span>:
        <span class="hljs-keyword">return</span> observation.tests_passed / <span class="hljs-built_in">max</span>(observation.tests_total, <span class="hljs-number">1</span>)


<span class="hljs-keyword">class</span> <span class="hljs-title class_">StyleRubric</span>(<span class="hljs-title class_ inherited__">Rubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, action, observation</span>) -&gt; <span class="hljs-built_in">float</span>:
        <span class="hljs-keyword">return</span> <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> action.code.count(<span class="hljs-string">&quot;\\n\\n\\n&quot;</span>) == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0.6</span>


reward = WeightedSum(
    [TestsPassRubric(), StyleRubric()],
    weights=[<span class="hljs-number">0.7</span>, <span class="hljs-number">0.3</span>],
)`,lang:"python",wrap:!1}}),L=new w({props:{title:"Gate — hard constraints",local:"gate--hard-constraints",headingTag:"h3"}}),F=new u({props:{code:"ZnJvbSUyMG9wZW5lbnYuY29yZS5ydWJyaWNzJTIwaW1wb3J0JTIwR2F0ZSUwQSUwQXJld2FyZCUyMCUzRCUyMEdhdGUoVGVzdHNQYXNzUnVicmljKCklMkMlMjB0aHJlc2hvbGQlM0QwLjUpJTIwJTIwJTIzJTIwMC4wJTIwaWYlMjBmZXdlciUyMHRoYW4lMjBoYWxmJTIwdGhlJTIwdGVzdHMlMjBwYXNz",highlighted:`<span class="hljs-keyword">from</span> openenv.core.rubrics <span class="hljs-keyword">import</span> Gate

reward = Gate(TestsPassRubric(), threshold=<span class="hljs-number">0.5</span>)  <span class="hljs-comment"># 0.0 if fewer than half the tests pass</span>`,lang:"python",wrap:!1}}),q=new w({props:{title:"Sequential — fail-fast pipeline",local:"sequential--fail-fast-pipeline",headingTag:"h3"}}),K=new u({props:{code:"ZnJvbSUyMG9wZW5lbnYuY29yZS5ydWJyaWNzJTIwaW1wb3J0JTIwU2VxdWVudGlhbCUwQSUwQXJld2FyZCUyMCUzRCUyMFNlcXVlbnRpYWwoJTBBJTIwJTIwJTIwJTIwR2F0ZShDb21waWxlc1J1YnJpYygpJTJDJTIwdGhyZXNob2xkJTNEMS4wKSUyQyUyMCUyMCUyMCUyMyUyMHNraXAlMjBldmVyeXRoaW5nJTIwaWYlMjBpdCUyMGRvZXNuJ3QlMjBjb21waWxlJTBBJTIwJTIwJTIwJTIwR2F0ZShUZXN0c1Bhc3NSdWJyaWMoKSUyQyUyMHRocmVzaG9sZCUzRDAuNSklMkMlMjAlMjAlMjMlMjBhbmQlMjBza2lwJTIwc3R5bGUlMjBpZiUyMHRlc3RzJTIwYXJlJTIwZmFpbGluZyUwQSUyMCUyMCUyMCUyMFdlaWdodGVkU3VtKCU1QlRlc3RzUGFzc1J1YnJpYygpJTJDJTIwU3R5bGVSdWJyaWMoKSU1RCUyQyUyMCU1QjAuNyUyQyUyMDAuMyU1RCklMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> openenv.core.rubrics <span class="hljs-keyword">import</span> Sequential

reward = Sequential(
    Gate(CompilesRubric(), threshold=<span class="hljs-number">1.0</span>),   <span class="hljs-comment"># skip everything if it doesn&#x27;t compile</span>
    Gate(TestsPassRubric(), threshold=<span class="hljs-number">0.5</span>),  <span class="hljs-comment"># and skip style if tests are failing</span>
    WeightedSum([TestsPassRubric(), StyleRubric()], [<span class="hljs-number">0.7</span>, <span class="hljs-number">0.3</span>]),
)`,lang:"python",wrap:!1}}),O=new w({props:{title:"RubricList and RubricDict — dynamic dispatch",local:"rubriclist-and-rubricdict--dynamic-dispatch",headingTag:"h3"}}),se=new u({props:{code:"ZnJvbSUyMG9wZW5lbnYuY29yZS5ydWJyaWNzJTIwaW1wb3J0JTIwUnVicmljJTJDJTIwUnVicmljRGljdCUwQSUwQSUwQWNsYXNzJTIwTXVsdGlHYW1lUnVicmljKFJ1YnJpYyklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN1cGVyKCkuX19pbml0X18oKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuZ2FtZXMlMjAlM0QlMjBSdWJyaWNEaWN0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBvbmclMjIlM0ElMjBQb25nUnVicmljKCklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJicmVha291dCUyMiUzQSUyMEJyZWFrb3V0UnVicmljKCklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QpJTBBJTBBJTIwJTIwJTIwJTIwZGVmJTIwZm9yd2FyZChzZWxmJTJDJTIwYWN0aW9uJTJDJTIwb2JzZXJ2YXRpb24pJTIwLSUzRSUyMGZsb2F0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2VsZi5nYW1lcyU1Qm9ic2VydmF0aW9uLmdhbWVfaWQlNUQoYWN0aW9uJTJDJTIwb2JzZXJ2YXRpb24p",highlighted:`<span class="hljs-keyword">from</span> openenv.core.rubrics <span class="hljs-keyword">import</span> Rubric, RubricDict


<span class="hljs-keyword">class</span> <span class="hljs-title class_">MultiGameRubric</span>(<span class="hljs-title class_ inherited__">Rubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.games = RubricDict({
            <span class="hljs-string">&quot;pong&quot;</span>: PongRubric(),
            <span class="hljs-string">&quot;breakout&quot;</span>: BreakoutRubric(),
        })

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, action, observation</span>) -&gt; <span class="hljs-built_in">float</span>:
        <span class="hljs-keyword">return</span> self.games[observation.game_id](action, observation)`,lang:"python",wrap:!1}}),te=new w({props:{title:"Introspection: named_rubrics()",local:"introspection-namedrubrics",headingTag:"h3"}}),ae=new u({props:{code:"Y29tcG9zaXRlJTIwJTNEJTIwV2VpZ2h0ZWRTdW0oJTBBJTIwJTIwJTIwJTIwJTVCR2F0ZShDb21waWxlc1J1YnJpYygpJTJDJTIwMS4wKSUyQyUyMFRlc3RzUGFzc1J1YnJpYygpJTJDJTIwU3R5bGVSdWJyaWMoKSU1RCUyQyUwQSUyMCUyMCUyMCUyMCU1QjAuMiUyQyUyMDAuNSUyQyUyMDAuMyU1RCUyQyUwQSklMEElMEFmb3IlMjBuYW1lJTJDJTIwY2hpbGQlMjBpbiUyMGNvbXBvc2l0ZS5uYW1lZF9ydWJyaWNzKCklM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyJTdCbmFtZSUzQTMwcyU3RCUyMGxhc3Rfc2NvcmUlM0QlN0JjaGlsZC5sYXN0X3Njb3JlJTdEJTIyKQ==",highlighted:`composite = WeightedSum(
    [Gate(CompilesRubric(), <span class="hljs-number">1.0</span>), TestsPassRubric(), StyleRubric()],
    [<span class="hljs-number">0.2</span>, <span class="hljs-number">0.5</span>, <span class="hljs-number">0.3</span>],
)

<span class="hljs-keyword">for</span> name, child <span class="hljs-keyword">in</span> composite.named_rubrics():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{name:30s}</span> last_score=<span class="hljs-subst">{child.last_score}</span>&quot;</span>)`,lang:"python",wrap:!1}}),re=new w({props:{title:"LLM-as-judge: LLMJudge",local:"llm-as-judge-llmjudge",headingTag:"h2"}}),pe=new u({props:{code:"aW1wb3J0JTIwb3MlMEElMEFmcm9tJTIwb3BlbmVudi5jb3JlLmxsbV9jbGllbnQlMjBpbXBvcnQlMjBPcGVuQUlDbGllbnQlMkMlMjBjcmVhdGVfbGxtX2NsaWVudCUwQWZyb20lMjBvcGVuZW52LmNvcmUucnVicmljcyUyMGltcG9ydCUyMExMTUp1ZGdlJTBBJTBBJTIzJTIwT3B0aW9uJTIwMSUyMCVFMiU4MCU5NCUyMGhvc3RlZCUyME9wZW5BSSUyMCh0aGUlMjBmYWN0b3J5JTIwYWxzbyUyMHN1cHBvcnRzJTIwJTIyYW50aHJvcGljJTIyKS4lMEFjbGllbnQlMjAlM0QlMjBjcmVhdGVfbGxtX2NsaWVudCglMEElMjAlMjAlMjAlMjAlMjJvcGVuYWklMjIlMkMlMEElMjAlMjAlMjAlMjBtb2RlbCUzRCUyMmdwdC00LjEtbWluaSUyMiUyQyUwQSUyMCUyMCUyMCUyMGFwaV9rZXklM0Rvcy5lbnZpcm9uJTVCJTIyT1BFTkFJX0FQSV9LRVklMjIlNUQlMkMlMEEpJTBBJTBBJTIzJTIwT3B0aW9uJTIwMiUyMCVFMiU4MCU5NCUyMG9wZW4td2VpZ2h0JTIwbW9kZWwlMjBzZXJ2ZWQlMjB2aWElMjBhJTIwbG9jYWwlMjBPcGVuQUktY29tcGF0aWJsZSUyMGVuZHBvaW50JTBBJTIzJTIwKHZMTE0lMkMlMjBPbGxhbWElMkMlMjBIdWdnaW5nJTIwRmFjZSUyMEluZmVyZW5jZSUyMFByb3ZpZGVycyUyQyUyMCVFMiU4MCVBNikuJTIwUG9pbnQlMjBPcGVuQUlDbGllbnQlMEElMjMlMjBhdCUyMHRoZSUyMGJhc2UlMjBVUkwlMjBhbmQlMjB0aGUlMjBtb2RlbCUyMGlkJTIwdGhlJTIwc2VydmVyJTIwZXhwb3Nlcy4lMjAlNjBhcGlfa2V5JTYwJTIwaXMlMjBvcHRpb25hbCUwQSUyMyUyMGFuZCUyMGRlZmF1bHRzJTIwdG8lMjAlMjJub3QtbmVlZGVkJTIyJTIwZm9yJTIwbG9jYWwlMjBlbmRwb2ludHMuJTBBY2xpZW50JTIwJTNEJTIwT3BlbkFJQ2xpZW50KCUwQSUyMCUyMCUyMCUyMGVuZHBvaW50JTNEJTIyaHR0cCUzQSUyRiUyRmxvY2FsaG9zdCUyMiUyQyUwQSUyMCUyMCUyMCUyMHBvcnQlM0Q4MDAwJTJDJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJRd2VuJTJGUXdlbjMtMS43QiUyMiUyQyUwQSklMEElMEFjbGFyaXR5X2p1ZGdlJTIwJTNEJTIwTExNSnVkZ2UoJTBBJTIwJTIwJTIwJTIwY2xpZW50JTNEY2xpZW50JTJDJTBBJTIwJTIwJTIwJTIwcHJvbXB0X3RlbXBsYXRlJTNEKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMlJhdGUlMjB0aGUlMjBjbGFyaXR5JTIwb2YlMjB0aGlzJTIwZXhwbGFuYXRpb24lMjBvbiUyMGElMjAwLTEwJTIwc2NhbGUuJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyUmVwbHklMjB3aXRoJTIwdGhlJTIwbnVtYmVyJTIwb25seS4lNUNuJTVDbiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMkV4cGxhbmF0aW9uJTNBJTVDbiU3QmFjdGlvbiU3RCU1Q24lMjIlMEElMjAlMjAlMjAlMjApJTJDJTBBJTIwJTIwJTIwJTIwc2NvcmVfcGF0dGVybiUzRHIlMjIoJTVDZCUyQiglM0YlM0ElNUMuJTVDZCUyQiklM0YpJTIyJTJDJTBBJTIwJTIwJTIwJTIwbm9ybWFsaXplJTNEVHJ1ZSUyQyUyMCUyMCUyMCUyMyUyMGNsYW1wcyUyMGV4dHJhY3RlZCUyMHNjb3JlJTIwdG8lMjAlNUIwJTJDJTIwMSU1RCUwQSk=",highlighted:`<span class="hljs-keyword">import</span> os

<span class="hljs-keyword">from</span> openenv.core.llm_client <span class="hljs-keyword">import</span> OpenAIClient, create_llm_client
<span class="hljs-keyword">from</span> openenv.core.rubrics <span class="hljs-keyword">import</span> LLMJudge

<span class="hljs-comment"># Option 1 — hosted OpenAI (the factory also supports &quot;anthropic&quot;).</span>
client = create_llm_client(
    <span class="hljs-string">&quot;openai&quot;</span>,
    model=<span class="hljs-string">&quot;gpt-4.1-mini&quot;</span>,
    api_key=os.environ[<span class="hljs-string">&quot;OPENAI_API_KEY&quot;</span>],
)

<span class="hljs-comment"># Option 2 — open-weight model served via a local OpenAI-compatible endpoint</span>
<span class="hljs-comment"># (vLLM, Ollama, Hugging Face Inference Providers, …). Point OpenAIClient</span>
<span class="hljs-comment"># at the base URL and the model id the server exposes. \`api_key\` is optional</span>
<span class="hljs-comment"># and defaults to &quot;not-needed&quot; for local endpoints.</span>
client = OpenAIClient(
    endpoint=<span class="hljs-string">&quot;http://localhost&quot;</span>,
    port=<span class="hljs-number">8000</span>,
    model=<span class="hljs-string">&quot;Qwen/Qwen3-1.7B&quot;</span>,
)

clarity_judge = LLMJudge(
    client=client,
    prompt_template=(
        <span class="hljs-string">&quot;Rate the clarity of this explanation on a 0-10 scale. &quot;</span>
        <span class="hljs-string">&quot;Reply with the number only.\\n\\n&quot;</span>
        <span class="hljs-string">&quot;Explanation:\\n{action}\\n&quot;</span>
    ),
    score_pattern=<span class="hljs-string">r&quot;(\\d+(?:\\.\\d+)?)&quot;</span>,
    normalize=<span class="hljs-literal">True</span>,   <span class="hljs-comment"># clamps extracted score to [0, 1]</span>
)`,lang:"python",wrap:!1}}),ye=new w({props:{title:"Delayed Rewards: TrajectoryRubric",local:"delayed-rewards-trajectoryrubric",headingTag:"h2"}}),we=new u({props:{code:"ZnJvbSUyMG9wZW5lbnYuY29yZS5ydWJyaWNzJTIwaW1wb3J0JTIwVHJhamVjdG9yeVJ1YnJpYyUwQSUwQSUwQWNsYXNzJTIwV2luTG9zc1J1YnJpYyhUcmFqZWN0b3J5UnVicmljKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMHNjb3JlX3RyYWplY3Rvcnkoc2VsZiUyQyUyMHRyYWplY3RvcnkpJTIwLSUzRSUyMGZsb2F0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwXyUyQyUyMGZpbmFsX29icyUyMCUzRCUyMHRyYWplY3RvcnklNUItMSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMGZpbmFsX29icy5yZXdhcmQlMjAlMjAlMjAlMjMlMjAlMkIxJTIwd2luJTJDJTIwLTElMjBsb3NzJTJDJTIwMCUyMGRyYXclMEElMEElMjAlMjAlMjAlMjBkZWYlMjBjb21wdXRlX3N0ZXBfcmV3YXJkcyhzZWxmKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMENyZWRpdCUyMGFzc2lnbm1lbnQlM0ElMjBkaXN0cmlidXRlJTIwdGhlJTIwZmluYWwlMjBzY29yZSUyMGFjcm9zcyUyMHN0ZXBzJTIwaG93ZXZlciUyMHlvdSUyMGxpa2UuJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmluYWwlMjAlM0QlMjBzZWxmLnNjb3JlX3RyYWplY3Rvcnkoc2VsZi5fdHJhamVjdG9yeSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmaW5hbCU1RCUyMColMjBsZW4oc2VsZi5fdHJhamVjdG9yeSk=",highlighted:`<span class="hljs-keyword">from</span> openenv.core.rubrics <span class="hljs-keyword">import</span> TrajectoryRubric


<span class="hljs-keyword">class</span> <span class="hljs-title class_">WinLossRubric</span>(<span class="hljs-title class_ inherited__">TrajectoryRubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">score_trajectory</span>(<span class="hljs-params">self, trajectory</span>) -&gt; <span class="hljs-built_in">float</span>:
        _, final_obs = trajectory[-<span class="hljs-number">1</span>]
        <span class="hljs-keyword">return</span> final_obs.reward   <span class="hljs-comment"># +1 win, -1 loss, 0 draw</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_step_rewards</span>(<span class="hljs-params">self</span>):
        <span class="hljs-comment"># Credit assignment: distribute the final score across steps however you like.</span>
        final = self.score_trajectory(self._trajectory)
        <span class="hljs-keyword">return</span> [final] * <span class="hljs-built_in">len</span>(self._trajectory)`,lang:"python",wrap:!1}}),he=new u({props:{code:"ZnJvbSUyMG9wZW5lbnYuY29yZS5ydWJyaWNzJTIwaW1wb3J0JTIwRXhwb25lbnRpYWxEaXNjb3VudGluZ1RyYWplY3RvcnlSdWJyaWMlMEElMEElMEFjbGFzcyUyMENoZXNzT3V0Y29tZVJ1YnJpYyhFeHBvbmVudGlhbERpc2NvdW50aW5nVHJhamVjdG9yeVJ1YnJpYyklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBzY29yZV90cmFqZWN0b3J5KHNlbGYlMkMlMjB0cmFqZWN0b3J5KSUyMC0lM0UlMjBmbG9hdCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMF8lMkMlMjBmaW5hbF9vYnMlMjAlM0QlMjB0cmFqZWN0b3J5JTVCLTElNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBmaW5hbF9vYnMucmV3YXJkJTIwJTIwJTIwJTIwJTIzJTIwYWxyZWFkeSUyMCUyQjElMjAlMkYlMjAwJTIwJTJGJTIwLTElMjBmcm9tJTIwdGhlJTIwZW5naW5l",highlighted:`<span class="hljs-keyword">from</span> openenv.core.rubrics <span class="hljs-keyword">import</span> ExponentialDiscountingTrajectoryRubric


<span class="hljs-keyword">class</span> <span class="hljs-title class_">ChessOutcomeRubric</span>(<span class="hljs-title class_ inherited__">ExponentialDiscountingTrajectoryRubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">score_trajectory</span>(<span class="hljs-params">self, trajectory</span>) -&gt; <span class="hljs-built_in">float</span>:
        _, final_obs = trajectory[-<span class="hljs-number">1</span>]
        <span class="hljs-keyword">return</span> final_obs.reward    <span class="hljs-comment"># already +1 / 0 / -1 from the engine</span>`,lang:"python",wrap:!1}}),Te=new w({props:{title:"Wiring a Rubric into an Environment",local:"wiring-a-rubric-into-an-environment",headingTag:"h2"}}),fe=new u({props:{code:"ZnJvbSUyMG9wZW5lbnYuY29yZS5lbnZfc2VydmVyLmludGVyZmFjZXMlMjBpbXBvcnQlMjBFbnZpcm9ubWVudCUwQWZyb20lMjBvcGVuZW52LmNvcmUuZW52X3NlcnZlci50eXBlcyUyMGltcG9ydCUyMEFjdGlvbiUyQyUyME9ic2VydmF0aW9uJTJDJTIwU3RhdGUlMEFmcm9tJTIwb3BlbmVudi5jb3JlLnJ1YnJpY3MlMjBpbXBvcnQlMjBHYXRlJTJDJTIwUnVicmljJTJDJTIwU2VxdWVudGlhbCUyQyUyMFdlaWdodGVkU3VtJTBBJTBBJTBBY2xhc3MlMjBDb2RlQWN0aW9uKEFjdGlvbiklM0ElMEElMjAlMjAlMjAlMjBjb2RlJTNBJTIwc3RyJTBBJTBBJTBBY2xhc3MlMjBDb2RlT2JzZXJ2YXRpb24oT2JzZXJ2YXRpb24pJTNBJTBBJTIwJTIwJTIwJTIwY29tcGlsZXMlM0ElMjBib29sJTIwJTNEJTIwRmFsc2UlMEElMjAlMjAlMjAlMjB0ZXN0c19wYXNzZWQlM0ElMjBpbnQlMjAlM0QlMjAwJTBBJTIwJTIwJTIwJTIwdGVzdHNfdG90YWwlM0ElMjBpbnQlMjAlM0QlMjAwJTBBJTBBJTBBY2xhc3MlMjBDb2RlU3RhdGUoU3RhdGUpJTNBJTBBJTIwJTIwJTIwJTIwYXR0ZW1wdHMlM0ElMjBpbnQlMjAlM0QlMjAwJTBBJTBBJTBBY2xhc3MlMjBDb21waWxlc1J1YnJpYyhSdWJyaWMpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwZm9yd2FyZChzZWxmJTJDJTIwYWN0aW9uJTJDJTIwb2JzZXJ2YXRpb24pJTIwLSUzRSUyMGZsb2F0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwMS4wJTIwaWYlMjBvYnNlcnZhdGlvbi5jb21waWxlcyUyMGVsc2UlMjAwLjAlMEElMEElMEFjbGFzcyUyMFRlc3RzUGFzc1J1YnJpYyhSdWJyaWMpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwZm9yd2FyZChzZWxmJTJDJTIwYWN0aW9uJTJDJTIwb2JzZXJ2YXRpb24pJTIwLSUzRSUyMGZsb2F0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBvYnNlcnZhdGlvbi50ZXN0c190b3RhbCUyMCUzRCUzRCUyMDAlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBvYnNlcnZhdGlvbi50ZXN0c19wYXNzZWQlMjAlMkYlMjBvYnNlcnZhdGlvbi50ZXN0c190b3RhbCUwQSUwQSUwQWNsYXNzJTIwU3R5bGVSdWJyaWMoUnVicmljKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMGFjdGlvbiUyQyUyMG9ic2VydmF0aW9uKSUyMC0lM0UlMjBmbG9hdCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMDEuMCUyMGlmJTIwYWN0aW9uLmNvZGUuY291bnQoJTIyJTVDbiU1Q24lNUNuJTIyKSUyMCUzRCUzRCUyMDAlMjBlbHNlJTIwMC42JTBBJTBBJTBBZGVmJTIwYnVpbGRfY29kZV9ydWJyaWMoKSUyMC0lM0UlMjBSdWJyaWMlM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBTZXF1ZW50aWFsKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEdhdGUoQ29tcGlsZXNSdWJyaWMoKSUyQyUyMHRocmVzaG9sZCUzRDEuMCklMkMlMjAlMjAlMjMlMjBnYXRlJTIwZXZlcnl0aGluZyUyMG9uJTIwY29tcGlsYXRpb24lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBXZWlnaHRlZFN1bSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBUZXN0c1Bhc3NSdWJyaWMoKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFN0eWxlUnVicmljKCklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3ZWlnaHRzJTNEJTVCMC43JTJDJTIwMC4zJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUyQyUwQSUyMCUyMCUyMCUyMCklMEElMEElMEFjbGFzcyUyMENvZGVFbnZpcm9ubWVudChFbnZpcm9ubWVudCU1QkNvZGVBY3Rpb24lMkMlMjBDb2RlT2JzZXJ2YXRpb24lMkMlMjBDb2RlU3RhdGUlNUQpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwX19pbml0X18oc2VsZiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdXBlcigpLl9faW5pdF9fKHJ1YnJpYyUzRGJ1aWxkX2NvZGVfcnVicmljKCkpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5fc3RhdGUlMjAlM0QlMjBDb2RlU3RhdGUoKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMHJlc2V0KHNlbGYlMkMlMjBzZWVkJTNETm9uZSUyQyUyMGVwaXNvZGVfaWQlM0ROb25lJTJDJTIwKiprd2FyZ3MpJTIwLSUzRSUyMENvZGVPYnNlcnZhdGlvbiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuX3Jlc2V0X3J1YnJpYygpJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwY2xlYXIlMjBhbnklMjB0cmFqZWN0b3J5JTIwJTJGJTIwY2FjaGVkJTIwbGFzdF9zY29yZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuX3N0YXRlJTIwJTNEJTIwQ29kZVN0YXRlKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBDb2RlT2JzZXJ2YXRpb24oKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMHN0ZXAoc2VsZiUyQyUyMGFjdGlvbiUzQSUyMENvZGVBY3Rpb24lMkMlMjB0aW1lb3V0X3MlM0ROb25lJTJDJTIwKiprd2FyZ3MpJTIwLSUzRSUyMENvZGVPYnNlcnZhdGlvbiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuX3N0YXRlLmF0dGVtcHRzJTIwJTJCJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9icyUyMCUzRCUyMHNlbGYuX3J1bl9jb2RlKGFjdGlvbiklMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjB5b3VyJTIwZG9tYWluLXNwZWNpZmljJTIwZXhlY3V0aW9uJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb2JzLnJld2FyZCUyMCUzRCUyMHNlbGYuX2FwcGx5X3J1YnJpYyhhY3Rpb24lMkMlMjBvYnMpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwb2JzJTBBJTBBJTIwJTIwJTIwJTIwJTQwcHJvcGVydHklMEElMjAlMjAlMjAlMjBkZWYlMjBzdGF0ZShzZWxmKSUyMC0lM0UlMjBDb2RlU3RhdGUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBzZWxmLl9zdGF0ZSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9ydW5fY29kZShzZWxmJTJDJTIwYWN0aW9uJTNBJTIwQ29kZUFjdGlvbiklMjAtJTNFJTIwQ29kZU9ic2VydmF0aW9uJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwUGxhY2Vob2xkZXIlMjBmb3IlMjB3aGF0ZXZlciUyMHlvdXIlMjBlbnZpcm9ubWVudCUyMGFjdHVhbGx5JTIwZG9lcy4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb21waWxlcyUyMCUzRCUyMCUyMmRlZiUyMCUyMiUyMGluJTIwYWN0aW9uLmNvZGUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBDb2RlT2JzZXJ2YXRpb24oJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29tcGlsZXMlM0Rjb21waWxlcyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRlc3RzX3Bhc3NlZCUzRDMlMjBpZiUyMGNvbXBpbGVzJTIwZWxzZSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0ZXN0c190b3RhbCUzRDMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAp",highlighted:`<span class="hljs-keyword">from</span> openenv.core.env_server.interfaces <span class="hljs-keyword">import</span> Environment
<span class="hljs-keyword">from</span> openenv.core.env_server.types <span class="hljs-keyword">import</span> Action, Observation, State
<span class="hljs-keyword">from</span> openenv.core.rubrics <span class="hljs-keyword">import</span> Gate, Rubric, Sequential, WeightedSum


<span class="hljs-keyword">class</span> <span class="hljs-title class_">CodeAction</span>(<span class="hljs-title class_ inherited__">Action</span>):
    code: <span class="hljs-built_in">str</span>


<span class="hljs-keyword">class</span> <span class="hljs-title class_">CodeObservation</span>(<span class="hljs-title class_ inherited__">Observation</span>):
    compiles: <span class="hljs-built_in">bool</span> = <span class="hljs-literal">False</span>
    tests_passed: <span class="hljs-built_in">int</span> = <span class="hljs-number">0</span>
    tests_total: <span class="hljs-built_in">int</span> = <span class="hljs-number">0</span>


<span class="hljs-keyword">class</span> <span class="hljs-title class_">CodeState</span>(<span class="hljs-title class_ inherited__">State</span>):
    attempts: <span class="hljs-built_in">int</span> = <span class="hljs-number">0</span>


<span class="hljs-keyword">class</span> <span class="hljs-title class_">CompilesRubric</span>(<span class="hljs-title class_ inherited__">Rubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, action, observation</span>) -&gt; <span class="hljs-built_in">float</span>:
        <span class="hljs-keyword">return</span> <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> observation.compiles <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span>


<span class="hljs-keyword">class</span> <span class="hljs-title class_">TestsPassRubric</span>(<span class="hljs-title class_ inherited__">Rubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, action, observation</span>) -&gt; <span class="hljs-built_in">float</span>:
        <span class="hljs-keyword">if</span> observation.tests_total == <span class="hljs-number">0</span>:
            <span class="hljs-keyword">return</span> <span class="hljs-number">0.0</span>
        <span class="hljs-keyword">return</span> observation.tests_passed / observation.tests_total


<span class="hljs-keyword">class</span> <span class="hljs-title class_">StyleRubric</span>(<span class="hljs-title class_ inherited__">Rubric</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, action, observation</span>) -&gt; <span class="hljs-built_in">float</span>:
        <span class="hljs-keyword">return</span> <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> action.code.count(<span class="hljs-string">&quot;\\n\\n\\n&quot;</span>) == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0.6</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">build_code_rubric</span>() -&gt; Rubric:
    <span class="hljs-keyword">return</span> Sequential(
        Gate(CompilesRubric(), threshold=<span class="hljs-number">1.0</span>),  <span class="hljs-comment"># gate everything on compilation</span>
        WeightedSum(
            [
                TestsPassRubric(),
                StyleRubric(),
            ],
            weights=[<span class="hljs-number">0.7</span>, <span class="hljs-number">0.3</span>],
        ),
    )


<span class="hljs-keyword">class</span> <span class="hljs-title class_">CodeEnvironment</span>(Environment[CodeAction, CodeObservation, CodeState]):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__(rubric=build_code_rubric())
        self._state = CodeState()

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, seed=<span class="hljs-literal">None</span>, episode_id=<span class="hljs-literal">None</span>, **kwargs</span>) -&gt; CodeObservation:
        self._reset_rubric()               <span class="hljs-comment"># clear any trajectory / cached last_score</span>
        self._state = CodeState()
        <span class="hljs-keyword">return</span> CodeObservation()

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">step</span>(<span class="hljs-params">self, action: CodeAction, timeout_s=<span class="hljs-literal">None</span>, **kwargs</span>) -&gt; CodeObservation:
        self._state.attempts += <span class="hljs-number">1</span>
        obs = self._run_code(action)       <span class="hljs-comment"># your domain-specific execution</span>
        obs.reward = self._apply_rubric(action, obs)
        <span class="hljs-keyword">return</span> obs

<span class="hljs-meta">    @property</span>
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">state</span>(<span class="hljs-params">self</span>) -&gt; CodeState:
        <span class="hljs-keyword">return</span> self._state

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">_run_code</span>(<span class="hljs-params">self, action: CodeAction</span>) -&gt; CodeObservation:
        <span class="hljs-comment"># Placeholder for whatever your environment actually does.</span>
        compiles = <span class="hljs-string">&quot;def &quot;</span> <span class="hljs-keyword">in</span> action.code
        <span class="hljs-keyword">return</span> CodeObservation(
            compiles=compiles,
            tests_passed=<span class="hljs-number">3</span> <span class="hljs-keyword">if</span> compiles <span class="hljs-keyword">else</span> <span class="hljs-number">0</span>,
            tests_total=<span class="hljs-number">3</span>,
        )`,lang:"python",wrap:!1}}),ge=new w({props:{title:"Inspecting rewards from training code",local:"inspecting-rewards-from-training-code",headingTag:"h3"}}),Be=new u({props:{code:"ZW52JTIwJTNEJTIwQ29kZUVudmlyb25tZW50KCklMEFvYnMlMjAlM0QlMjBlbnYucmVzZXQoKSUwQW9icyUyMCUzRCUyMGVudi5zdGVwKENvZGVBY3Rpb24oY29kZSUzRCUyMmRlZiUyMHNvbHV0aW9uKCklM0ElMjByZXR1cm4lMjA0MiUyMikpJTBBJTBBZm9yJTIwbmFtZSUyQyUyMGNvbXBvbmVudCUyMGluJTIwZW52LnJ1YnJpYy5uYW1lZF9ydWJyaWNzKCklM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyJTdCbmFtZSUzQTMwcyU3RCUyMGxhc3Rfc2NvcmUlM0QlN0Jjb21wb25lbnQubGFzdF9zY29yZSUzQS4yZiU3RCUyMik=",highlighted:`env = CodeEnvironment()
obs = env.reset()
obs = env.step(CodeAction(code=<span class="hljs-string">&quot;def solution(): return 42&quot;</span>))

<span class="hljs-keyword">for</span> name, component <span class="hljs-keyword">in</span> env.rubric.named_rubrics():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{name:30s}</span> last_score=<span class="hljs-subst">{component.last_score:<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,lang:"python",wrap:!1}}),Ze=new w({props:{title:"Where the reward ends up during training",local:"where-the-reward-ends-up-during-training",headingTag:"h3"}}),We=new w({props:{title:"Using Rubrics for Evaluation",local:"using-rubrics-for-evaluation",headingTag:"h2"}}),$e=new u({props:{code:"cnVicmljJTIwJTNEJTIwYnVpbGRfY29kZV9ydWJyaWMoKSUwQSUwQXNjb3JlcyUyMCUzRCUyMCU1QiU1RCUwQWZvciUyMGFjdGlvbiUyQyUyMG9icyUyMGluJTIwZXZhbF9kYXRhc2V0JTNBJTBBJTIwJTIwJTIwJTIwc2NvcmVzLmFwcGVuZChydWJyaWMoYWN0aW9uJTJDJTIwb2JzKSklMEElMEFwcmludChmJTIybWVhbiUyMHJld2FyZCUzQSUyMCU3QnN1bShzY29yZXMpJTIwJTJGJTIwbGVuKHNjb3JlcyklM0EuM2YlN0QlMjIpJTBBZm9yJTIwbmFtZSUyQyUyMGNvbXBvbmVudCUyMGluJTIwcnVicmljLm5hbWVkX3J1YnJpY3MoKSUzQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlMjAlMjAlN0JuYW1lJTNBMzBzJTdEJTIwbGFzdF9zY29yZSUzRCU3QmNvbXBvbmVudC5sYXN0X3Njb3JlJTNBLjNmJTdEJTIyKQ==",highlighted:`rubric = build_code_rubric()

scores = []
<span class="hljs-keyword">for</span> action, obs <span class="hljs-keyword">in</span> eval_dataset:
    scores.append(rubric(action, obs))

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;mean reward: <span class="hljs-subst">{<span class="hljs-built_in">sum</span>(scores) / <span class="hljs-built_in">len</span>(scores):<span class="hljs-number">.3</span>f}</span>&quot;</span>)
<span class="hljs-keyword">for</span> name, component <span class="hljs-keyword">in</span> rubric.named_rubrics():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;  <span class="hljs-subst">{name:30s}</span> last_score=<span class="hljs-subst">{component.last_score:<span class="hljs-number">.3</span>f}</span>&quot;</span>)`,lang:"python",wrap:!1}}),Ee=new w({props:{title:"Next Steps",local:"next-steps",headingTag:"h2"}}),Se=new wt({props:{source:"https://github.com/huggingface/openenv/blob/main/docs/source/tutorials/rubrics.md"}}),{c(){J=i("meta"),xe=n(),Qe=i("p"),ze=n(),c(b.$$.fragment),Le=n(),c(U.$$.fragment),He=n(),f=i("p"),f.innerHTML=Jl,Fe=n(),C=i("p"),C.textContent=hl,De=n(),c(I.$$.fragment),qe=n(),g=i("p"),g.textContent=ml,Pe=n(),v=i("ol"),v.innerHTML=Tl,Ke=n(),B=i("p"),B.innerHTML=bl,Oe=n(),c(A.$$.fragment),es=n(),Z=i("p"),Z.innerHTML=Ul,ss=n(),c(_.$$.fragment),ls=n(),k=i("p"),k.textContent=fl,ts=n(),c(R.$$.fragment),ns=n(),W=i("p"),W.innerHTML=Cl,as=n(),c(G.$$.fragment),is=n(),$=i("p"),$.innerHTML=Il,rs=n(),c(Y.$$.fragment),os=n(),c(E.$$.fragment),cs=n(),N=i("p"),N.innerHTML=gl,ps=n(),c(S.$$.fragment),Ms=n(),Q=i("p"),Q.innerHTML=vl,ds=n(),c(V.$$.fragment),ys=n(),X=i("p"),X.textContent=Bl,js=n(),c(x.$$.fragment),ws=n(),z=i("p"),z.innerHTML=Al,us=n(),c(L.$$.fragment),Js=n(),H=i("p"),H.textContent=Zl,hs=n(),c(F.$$.fragment),ms=n(),D=i("p"),D.innerHTML=_l,Ts=n(),c(q.$$.fragment),bs=n(),P=i("p"),P.innerHTML=kl,Us=n(),c(K.$$.fragment),fs=n(),c(O.$$.fragment),Cs=n(),ee=i("p"),ee.innerHTML=Rl,Is=n(),c(se.$$.fragment),gs=n(),le=i("p"),le.innerHTML=Wl,vs=n(),c(te.$$.fragment),Bs=n(),ne=i("p"),ne.textContent=Gl,As=n(),c(ae.$$.fragment),Zs=n(),ie=i("p"),ie.innerHTML=$l,_s=n(),c(re.$$.fragment),ks=n(),oe=i("p"),oe.innerHTML=Yl,Rs=n(),ce=i("p"),ce.innerHTML=El,Ws=n(),c(pe.$$.fragment),Gs=n(),Me=i("p"),Me.innerHTML=Nl,$s=n(),de=i("ul"),de.innerHTML=Sl,Ys=n(),c(ye.$$.fragment),Es=n(),je=i("p"),je.innerHTML=Ql,Ns=n(),c(we.$$.fragment),Ss=n(),ue=i("p"),ue.innerHTML=Vl,Qs=n(),h=i("blockquote"),h.innerHTML=Xl,Vs=n(),Je=i("p"),Je.innerHTML=xl,Xs=n(),c(he.$$.fragment),xs=n(),me=i("p"),me.innerHTML=zl,zs=n(),m=i("blockquote"),m.innerHTML=Ll,Ls=n(),c(Te.$$.fragment),Hs=n(),be=i("p"),be.innerHTML=Hl,Fs=n(),Ue=i("p"),Ue.innerHTML=Fl,Ds=n(),c(fe.$$.fragment),qs=n(),Ce=i("p"),Ce.textContent=Dl,Ps=n(),Ie=i("ol"),Ie.innerHTML=ql,Ks=n(),T=i("blockquote"),T.innerHTML=Pl,Os=n(),c(ge.$$.fragment),el=n(),ve=i("p"),ve.textContent=Kl,sl=n(),c(Be.$$.fragment),ll=n(),Ae=i("p"),Ae.innerHTML=Ol,tl=n(),c(Ze.$$.fragment),nl=n(),_e=i("p"),_e.innerHTML=et,al=n(),ke=i("p"),ke.innerHTML=st,il=n(),Re=i("p"),Re.innerHTML=lt,rl=n(),c(We.$$.fragment),ol=n(),Ge=i("p"),Ge.textContent=tt,cl=n(),c($e.$$.fragment),pl=n(),Ye=i("p"),Ye.innerHTML=nt,Ml=n(),c(Ee.$$.fragment),dl=n(),Ne=i("ul"),Ne.innerHTML=at,yl=n(),c(Se.$$.fragment),jl=n(),Ve=i("p"),this.h()},l(e){const s=dt("svelte-u9bgzb",document.head);J=r(s,"META",{name:!0,content:!0}),s.forEach(l),xe=a(e),Qe=r(e,"P",{}),it(Qe).forEach(l),ze=a(e),p(b.$$.fragment,e),Le=a(e),p(U.$$.fragment,e),He=a(e),f=r(e,"P",{"data-svelte-h":!0}),o(f)!=="svelte-1laso4n"&&(f.innerHTML=Jl),Fe=a(e),C=r(e,"P",{"data-svelte-h":!0}),o(C)!=="svelte-1xlkudf"&&(C.textContent=hl),De=a(e),p(I.$$.fragment,e),qe=a(e),g=r(e,"P",{"data-svelte-h":!0}),o(g)!=="svelte-f9hwlt"&&(g.textContent=ml),Pe=a(e),v=r(e,"OL",{"data-svelte-h":!0}),o(v)!=="svelte-1qwlix3"&&(v.innerHTML=Tl),Ke=a(e),B=r(e,"P",{"data-svelte-h":!0}),o(B)!=="svelte-xru7q2"&&(B.innerHTML=bl),Oe=a(e),p(A.$$.fragment,e),es=a(e),Z=r(e,"P",{"data-svelte-h":!0}),o(Z)!=="svelte-olozwl"&&(Z.innerHTML=Ul),ss=a(e),p(_.$$.fragment,e),ls=a(e),k=r(e,"P",{"data-svelte-h":!0}),o(k)!=="svelte-aunoh0"&&(k.textContent=fl),ts=a(e),p(R.$$.fragment,e),ns=a(e),W=r(e,"P",{"data-svelte-h":!0}),o(W)!=="svelte-1jq8fpu"&&(W.innerHTML=Cl),as=a(e),p(G.$$.fragment,e),is=a(e),$=r(e,"P",{"data-svelte-h":!0}),o($)!=="svelte-17klfxh"&&($.innerHTML=Il),rs=a(e),p(Y.$$.fragment,e),os=a(e),p(E.$$.fragment,e),cs=a(e),N=r(e,"P",{"data-svelte-h":!0}),o(N)!=="svelte-1bw6joe"&&(N.innerHTML=gl),ps=a(e),p(S.$$.fragment,e),Ms=a(e),Q=r(e,"P",{"data-svelte-h":!0}),o(Q)!=="svelte-klg2vy"&&(Q.innerHTML=vl),ds=a(e),p(V.$$.fragment,e),ys=a(e),X=r(e,"P",{"data-svelte-h":!0}),o(X)!=="svelte-lobvd6"&&(X.textContent=Bl),js=a(e),p(x.$$.fragment,e),ws=a(e),z=r(e,"P",{"data-svelte-h":!0}),o(z)!=="svelte-mhec84"&&(z.innerHTML=Al),us=a(e),p(L.$$.fragment,e),Js=a(e),H=r(e,"P",{"data-svelte-h":!0}),o(H)!=="svelte-bdmrwx"&&(H.textContent=Zl),hs=a(e),p(F.$$.fragment,e),ms=a(e),D=r(e,"P",{"data-svelte-h":!0}),o(D)!=="svelte-1yo4yv7"&&(D.innerHTML=_l),Ts=a(e),p(q.$$.fragment,e),bs=a(e),P=r(e,"P",{"data-svelte-h":!0}),o(P)!=="svelte-1cdt1bs"&&(P.innerHTML=kl),Us=a(e),p(K.$$.fragment,e),fs=a(e),p(O.$$.fragment,e),Cs=a(e),ee=r(e,"P",{"data-svelte-h":!0}),o(ee)!=="svelte-5pycgi"&&(ee.innerHTML=Rl),Is=a(e),p(se.$$.fragment,e),gs=a(e),le=r(e,"P",{"data-svelte-h":!0}),o(le)!=="svelte-i67p6v"&&(le.innerHTML=Wl),vs=a(e),p(te.$$.fragment,e),Bs=a(e),ne=r(e,"P",{"data-svelte-h":!0}),o(ne)!=="svelte-id7406"&&(ne.textContent=Gl),As=a(e),p(ae.$$.fragment,e),Zs=a(e),ie=r(e,"P",{"data-svelte-h":!0}),o(ie)!=="svelte-1lhf4ru"&&(ie.innerHTML=$l),_s=a(e),p(re.$$.fragment,e),ks=a(e),oe=r(e,"P",{"data-svelte-h":!0}),o(oe)!=="svelte-jvs3uj"&&(oe.innerHTML=Yl),Rs=a(e),ce=r(e,"P",{"data-svelte-h":!0}),o(ce)!=="svelte-156gl6o"&&(ce.innerHTML=El),Ws=a(e),p(pe.$$.fragment,e),Gs=a(e),Me=r(e,"P",{"data-svelte-h":!0}),o(Me)!=="svelte-1656tou"&&(Me.innerHTML=Nl),$s=a(e),de=r(e,"UL",{"data-svelte-h":!0}),o(de)!=="svelte-1hepnsr"&&(de.innerHTML=Sl),Ys=a(e),p(ye.$$.fragment,e),Es=a(e),je=r(e,"P",{"data-svelte-h":!0}),o(je)!=="svelte-etza88"&&(je.innerHTML=Ql),Ns=a(e),p(we.$$.fragment,e),Ss=a(e),ue=r(e,"P",{"data-svelte-h":!0}),o(ue)!=="svelte-at62ia"&&(ue.innerHTML=Vl),Qs=a(e),h=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(h)!=="svelte-1dmzgc1"&&(h.innerHTML=Xl),Vs=a(e),Je=r(e,"P",{"data-svelte-h":!0}),o(Je)!=="svelte-yf2if4"&&(Je.innerHTML=xl),Xs=a(e),p(he.$$.fragment,e),xs=a(e),me=r(e,"P",{"data-svelte-h":!0}),o(me)!=="svelte-1egxi4w"&&(me.innerHTML=zl),zs=a(e),m=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(m)!=="svelte-14pt5q7"&&(m.innerHTML=Ll),Ls=a(e),p(Te.$$.fragment,e),Hs=a(e),be=r(e,"P",{"data-svelte-h":!0}),o(be)!=="svelte-1wdgjtw"&&(be.innerHTML=Hl),Fs=a(e),Ue=r(e,"P",{"data-svelte-h":!0}),o(Ue)!=="svelte-d7nldd"&&(Ue.innerHTML=Fl),Ds=a(e),p(fe.$$.fragment,e),qs=a(e),Ce=r(e,"P",{"data-svelte-h":!0}),o(Ce)!=="svelte-gy5svv"&&(Ce.textContent=Dl),Ps=a(e),Ie=r(e,"OL",{"data-svelte-h":!0}),o(Ie)!=="svelte-1r3ijbt"&&(Ie.innerHTML=ql),Ks=a(e),T=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(T)!=="svelte-9vcx4d"&&(T.innerHTML=Pl),Os=a(e),p(ge.$$.fragment,e),el=a(e),ve=r(e,"P",{"data-svelte-h":!0}),o(ve)!=="svelte-1rkkkyv"&&(ve.textContent=Kl),sl=a(e),p(Be.$$.fragment,e),ll=a(e),Ae=r(e,"P",{"data-svelte-h":!0}),o(Ae)!=="svelte-kpezmh"&&(Ae.innerHTML=Ol),tl=a(e),p(Ze.$$.fragment,e),nl=a(e),_e=r(e,"P",{"data-svelte-h":!0}),o(_e)!=="svelte-1e8hy2q"&&(_e.innerHTML=et),al=a(e),ke=r(e,"P",{"data-svelte-h":!0}),o(ke)!=="svelte-orqibu"&&(ke.innerHTML=st),il=a(e),Re=r(e,"P",{"data-svelte-h":!0}),o(Re)!=="svelte-rubsu0"&&(Re.innerHTML=lt),rl=a(e),p(We.$$.fragment,e),ol=a(e),Ge=r(e,"P",{"data-svelte-h":!0}),o(Ge)!=="svelte-blel7i"&&(Ge.textContent=tt),cl=a(e),p($e.$$.fragment,e),pl=a(e),Ye=r(e,"P",{"data-svelte-h":!0}),o(Ye)!=="svelte-7mo41g"&&(Ye.innerHTML=nt),Ml=a(e),p(Ee.$$.fragment,e),dl=a(e),Ne=r(e,"UL",{"data-svelte-h":!0}),o(Ne)!=="svelte-wcm2j7"&&(Ne.innerHTML=at),yl=a(e),p(Se.$$.fragment,e),jl=a(e),Ve=r(e,"P",{}),it(Ve).forEach(l),this.h()},h(){Xe(J,"name","hf:doc:metadata"),Xe(J,"content",Jt),Xe(h,"class","caution"),Xe(m,"class","caution"),Xe(T,"class","note")},m(e,s){yt(document.head,J),t(e,xe,s),t(e,Qe,s),t(e,ze,s),M(b,e,s),t(e,Le,s),M(U,e,s),t(e,He,s),t(e,f,s),t(e,Fe,s),t(e,C,s),t(e,De,s),M(I,e,s),t(e,qe,s),t(e,g,s),t(e,Pe,s),t(e,v,s),t(e,Ke,s),t(e,B,s),t(e,Oe,s),M(A,e,s),t(e,es,s),t(e,Z,s),t(e,ss,s),M(_,e,s),t(e,ls,s),t(e,k,s),t(e,ts,s),M(R,e,s),t(e,ns,s),t(e,W,s),t(e,as,s),M(G,e,s),t(e,is,s),t(e,$,s),t(e,rs,s),M(Y,e,s),t(e,os,s),M(E,e,s),t(e,cs,s),t(e,N,s),t(e,ps,s),M(S,e,s),t(e,Ms,s),t(e,Q,s),t(e,ds,s),M(V,e,s),t(e,ys,s),t(e,X,s),t(e,js,s),M(x,e,s),t(e,ws,s),t(e,z,s),t(e,us,s),M(L,e,s),t(e,Js,s),t(e,H,s),t(e,hs,s),M(F,e,s),t(e,ms,s),t(e,D,s),t(e,Ts,s),M(q,e,s),t(e,bs,s),t(e,P,s),t(e,Us,s),M(K,e,s),t(e,fs,s),M(O,e,s),t(e,Cs,s),t(e,ee,s),t(e,Is,s),M(se,e,s),t(e,gs,s),t(e,le,s),t(e,vs,s),M(te,e,s),t(e,Bs,s),t(e,ne,s),t(e,As,s),M(ae,e,s),t(e,Zs,s),t(e,ie,s),t(e,_s,s),M(re,e,s),t(e,ks,s),t(e,oe,s),t(e,Rs,s),t(e,ce,s),t(e,Ws,s),M(pe,e,s),t(e,Gs,s),t(e,Me,s),t(e,$s,s),t(e,de,s),t(e,Ys,s),M(ye,e,s),t(e,Es,s),t(e,je,s),t(e,Ns,s),M(we,e,s),t(e,Ss,s),t(e,ue,s),t(e,Qs,s),t(e,h,s),t(e,Vs,s),t(e,Je,s),t(e,Xs,s),M(he,e,s),t(e,xs,s),t(e,me,s),t(e,zs,s),t(e,m,s),t(e,Ls,s),M(Te,e,s),t(e,Hs,s),t(e,be,s),t(e,Fs,s),t(e,Ue,s),t(e,Ds,s),M(fe,e,s),t(e,qs,s),t(e,Ce,s),t(e,Ps,s),t(e,Ie,s),t(e,Ks,s),t(e,T,s),t(e,Os,s),M(ge,e,s),t(e,el,s),t(e,ve,s),t(e,sl,s),M(Be,e,s),t(e,ll,s),t(e,Ae,s),t(e,tl,s),M(Ze,e,s),t(e,nl,s),t(e,_e,s),t(e,al,s),t(e,ke,s),t(e,il,s),t(e,Re,s),t(e,rl,s),M(We,e,s),t(e,ol,s),t(e,Ge,s),t(e,cl,s),M($e,e,s),t(e,pl,s),t(e,Ye,s),t(e,Ml,s),M(Ee,e,s),t(e,dl,s),t(e,Ne,s),t(e,yl,s),M(Se,e,s),t(e,jl,s),t(e,Ve,s),wl=!0},p:ot,i(e){wl||(d(b.$$.fragment,e),d(U.$$.fragment,e),d(I.$$.fragment,e),d(A.$$.fragment,e),d(_.$$.fragment,e),d(R.$$.fragment,e),d(G.$$.fragment,e),d(Y.$$.fragment,e),d(E.$$.fragment,e),d(S.$$.fragment,e),d(V.$$.fragment,e),d(x.$$.fragment,e),d(L.$$.fragment,e),d(F.$$.fragment,e),d(q.$$.fragment,e),d(K.$$.fragment,e),d(O.$$.fragment,e),d(se.$$.fragment,e),d(te.$$.fragment,e),d(ae.$$.fragment,e),d(re.$$.fragment,e),d(pe.$$.fragment,e),d(ye.$$.fragment,e),d(we.$$.fragment,e),d(he.$$.fragment,e),d(Te.$$.fragment,e),d(fe.$$.fragment,e),d(ge.$$.fragment,e),d(Be.$$.fragment,e),d(Ze.$$.fragment,e),d(We.$$.fragment,e),d($e.$$.fragment,e),d(Ee.$$.fragment,e),d(Se.$$.fragment,e),wl=!0)},o(e){y(b.$$.fragment,e),y(U.$$.fragment,e),y(I.$$.fragment,e),y(A.$$.fragment,e),y(_.$$.fragment,e),y(R.$$.fragment,e),y(G.$$.fragment,e),y(Y.$$.fragment,e),y(E.$$.fragment,e),y(S.$$.fragment,e),y(V.$$.fragment,e),y(x.$$.fragment,e),y(L.$$.fragment,e),y(F.$$.fragment,e),y(q.$$.fragment,e),y(K.$$.fragment,e),y(O.$$.fragment,e),y(se.$$.fragment,e),y(te.$$.fragment,e),y(ae.$$.fragment,e),y(re.$$.fragment,e),y(pe.$$.fragment,e),y(ye.$$.fragment,e),y(we.$$.fragment,e),y(he.$$.fragment,e),y(Te.$$.fragment,e),y(fe.$$.fragment,e),y(ge.$$.fragment,e),y(Be.$$.fragment,e),y(Ze.$$.fragment,e),y(We.$$.fragment,e),y($e.$$.fragment,e),y(Ee.$$.fragment,e),y(Se.$$.fragment,e),wl=!1},d(e){e&&(l(xe),l(Qe),l(ze),l(Le),l(He),l(f),l(Fe),l(C),l(De),l(qe),l(g),l(Pe),l(v),l(Ke),l(B),l(Oe),l(es),l(Z),l(ss),l(ls),l(k),l(ts),l(ns),l(W),l(as),l(is),l($),l(rs),l(os),l(cs),l(N),l(ps),l(Ms),l(Q),l(ds),l(ys),l(X),l(js),l(ws),l(z),l(us),l(Js),l(H),l(hs),l(ms),l(D),l(Ts),l(bs),l(P),l(Us),l(fs),l(Cs),l(ee),l(Is),l(gs),l(le),l(vs),l(Bs),l(ne),l(As),l(Zs),l(ie),l(_s),l(ks),l(oe),l(Rs),l(ce),l(Ws),l(Gs),l(Me),l($s),l(de),l(Ys),l(Es),l(je),l(Ns),l(Ss),l(ue),l(Qs),l(h),l(Vs),l(Je),l(Xs),l(xs),l(me),l(zs),l(m),l(Ls),l(Hs),l(be),l(Fs),l(Ue),l(Ds),l(qs),l(Ce),l(Ps),l(Ie),l(Ks),l(T),l(Os),l(el),l(ve),l(sl),l(ll),l(Ae),l(tl),l(nl),l(_e),l(al),l(ke),l(il),l(Re),l(rl),l(ol),l(Ge),l(cl),l(pl),l(Ye),l(Ml),l(dl),l(Ne),l(yl),l(jl),l(Ve)),l(J),j(b,e),j(U,e),j(I,e),j(A,e),j(_,e),j(R,e),j(G,e),j(Y,e),j(E,e),j(S,e),j(V,e),j(x,e),j(L,e),j(F,e),j(q,e),j(K,e),j(O,e),j(se,e),j(te,e),j(ae,e),j(re,e),j(pe,e),j(ye,e),j(we,e),j(he,e),j(Te,e),j(fe,e),j(ge,e),j(Be,e),j(Ze,e),j(We,e),j($e,e),j(Ee,e),j(Se,e)}}}const Jt='{"title":"Rubrics: Composable Reward Computation","local":"rubrics-composable-reward-computation","sections":[{"title":"Why Rubrics?","local":"why-rubrics","sections":[],"depth":2},{"title":"Your First Rubric","local":"your-first-rubric","sections":[{"title":"Optional hooks for observability","local":"optional-hooks-for-observability","sections":[],"depth":3},{"title":"State dict","local":"state-dict","sections":[],"depth":3}],"depth":2},{"title":"Composing Rubrics","local":"composing-rubrics","sections":[{"title":"WeightedSum — multi-criteria averaging","local":"weightedsum--multi-criteria-averaging","sections":[],"depth":3},{"title":"Gate — hard constraints","local":"gate--hard-constraints","sections":[],"depth":3},{"title":"Sequential — fail-fast pipeline","local":"sequential--fail-fast-pipeline","sections":[],"depth":3},{"title":"RubricList and RubricDict — dynamic dispatch","local":"rubriclist-and-rubricdict--dynamic-dispatch","sections":[],"depth":3},{"title":"Introspection: named_rubrics()","local":"introspection-namedrubrics","sections":[],"depth":3}],"depth":2},{"title":"LLM-as-judge: LLMJudge","local":"llm-as-judge-llmjudge","sections":[],"depth":2},{"title":"Delayed Rewards: TrajectoryRubric","local":"delayed-rewards-trajectoryrubric","sections":[],"depth":2},{"title":"Wiring a Rubric into an Environment","local":"wiring-a-rubric-into-an-environment","sections":[{"title":"Inspecting rewards from training code","local":"inspecting-rewards-from-training-code","sections":[],"depth":3},{"title":"Where the reward ends up during training","local":"where-the-reward-ends-up-during-training","sections":[],"depth":3}],"depth":2},{"title":"Using Rubrics for Evaluation","local":"using-rubrics-for-evaluation","sections":[],"depth":2},{"title":"Next Steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';function ht(ul){return ct(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ct extends pt{constructor(J){super(),Mt(this,J,ht,ut,rt,{})}}export{Ct as component};