openenv/pr_772/en/_app/immutable/nodes/59.bb5bbbca.js

import{s as js,n as Us,o as hs}from"../chunks/scheduler.2b22cead.js";import{S as ds,i as Is,e as M,s as a,c as i,h as ms,a as o,d as e,b as n,f as Ts,g as c,j as p,k as ql,l as us,m as t,n as r,t as y,o as w,p as J}from"../chunks/index.1a0e8013.js";import{C as bs,H as j}from"../chunks/Heading.44521446.js";import{C as d}from"../chunks/CodeBlock.9040b71c.js";import{E as Cs}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.c54858a8.js";function vs(xl){let T,nl,tl,Ml,I,ol,m,pl,u,Ll='<a href="https://colab.research.google.com/github/huggingface/OpenEnv/blob/main/examples/evaluation_inspect.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>',il,b,Ol=`After training a model in an OpenEnv environment, you need to measure how it
actually performs on a held-out set of episodes. OpenEnv integrates with
<a href="https://inspect.aisi.org.uk/" rel="nofollow">Inspect AI</a> — an open-source evaluation
framework by the UK AI Safety Institute — through <code>InspectAIHarness</code>.`,cl,C,rl,v,Pl="Inspect AI and OpenEnv are complementary, not overlapping:",yl,f,Dl=`<li><strong>OpenEnv</strong> provides the environment (reset, step, reward) and the training
infrastructure (GRPO via TRL).</li> <li><strong>Inspect AI</strong> provides the evaluation infrastructure: datasets, solvers,
scorers, and structured logs.</li>`,wl,B,Kl=`<code>InspectAIHarness</code> is the bridge. It wraps <code>inspect_ai.eval()</code> inside
OpenEnv’s <code>EvalHarness</code> interface so that eval runs are tracked with the same
structured <code>EvalConfig</code> / <code>EvalResult</code> types you use across all harnesses.`,Jl,A,ls="The typical workflow is:",Tl,k,jl,Z,Ul,g,hl,G,ss=`<code>inspect-ai</code> is an optional dependency — <code>InspectAIHarness</code> is importable
without it, but raises a clear <code>ImportError</code> at call time if it is missing.`,dl,E,Il,V,es=`Uncomment exactly one option. All three feed into the same task and harness —
no other cells need to change.`,ml,N,ul,X,ts=`The <code>model</code> string uses <code>provider/model-name</code> format for API providers.
For local models, the <code>hf/</code> prefix loads the model with <code>transformers</code> — point
it at a Hub ID to download, or a local path (<code>hf/./path/to/checkpoint</code>) to use
weights you already have on disk (e.g. from TRL training).`,bl,S,Cl,Q,as=`An Inspect AI <code>Task</code> has three parts: a <strong>dataset</strong> of samples to evaluate,
a <strong>solver</strong> that runs the model (and optionally the environment), and a
<strong>scorer</strong> that grades each sample.`,vl,R,ns=`The example below evaluates a model against <code>echo_env</code> — the reference
OpenEnv environment. The model is asked to repeat a phrase; the solver sends
the phrase to the environment and records the echoed response; the scorer
checks it matches the expected output.`,fl,W,Ms=`The solver calls Inspect AI’s <code>generate()</code> to get the model’s output, then
sends it to the environment. The dataset, scorer, and harness are identical
for both providers.`,Bl,Y,Al,U,os=`<p><code>echo_env</code> is a pure MCP environment. Interact with it via <code>MCPToolClient</code>
and <code>call_tool(&quot;echo_message&quot;, ...)</code>. For non-MCP environments, use
<code>GenericEnvClient</code> instead.</p>`,kl,_,Zl,$,ps=`Pass the task to <code>InspectAIHarness</code> via <code>EvalConfig</code>. The <code>task</code> key in
<code>eval_parameters</code> takes a task object or a registered task name string.`,gl,H,Gl,F,is=`The <code>EvalResult</code> carries both the config and the scores, making it easy to
log, compare across runs, or serialize to JSON:`,El,z,Vl,q,Nl,x,cs=`Inspect AI tasks can also be defined in standalone <code>.py</code> files and referenced
by path. This is useful for CI pipelines where the task definition lives in
the repo and the harness is called from a script:`,Xl,L,Sl,O,Ql,P,rs="Replace <code>echo_env_solver</code> with a solver that uses your env and model:",Rl,D,ys=`<li><strong>Dataset</strong> — collect held-out episodes from your env (or a static
benchmark); each <code>Sample</code> needs <code>input</code> and <code>target</code> fields.</li> <li><strong>Solver</strong> — call your trained model against the env via <code>generate()</code>.
If you used GRPO training with an <code>environment_factory</code>, reuse the same
factory here so the eval env matches training exactly.</li> <li><strong>Scorer</strong> — use the env’s reward signal directly, or write an Inspect AI
<code>@scorer</code> that checks the final observation against a ground-truth target.</li>`,Wl,h,ws=`<p>Run this eval <strong>before training</strong> on your base model to establish a baseline,
then again after training to measure the improvement. The delta (post − pre)
is more informative than either number alone — a model that scores 60% after
training tells you little without knowing it started at 4%.</p>`,Yl,K,_l,ll,$l,sl,Js=`<li><a href="end-to-end-walkthrough">End-to-end walkthrough</a> — full GRPO training loop that produces a model you can evaluate with this tutorial</li> <li><a href="sft-warmup">SFT warm-up tutorial</a> — collect rollouts, filter by reward, and fine-tune a student model before running GRPO</li> <li><a href="rubrics">Rubrics tutorial</a> — define reward functions inside
the environment using composable rubrics</li> <li><a href="https://inspect.aisi.org.uk/" rel="nofollow">Inspect AI documentation</a> — full reference
for tasks, solvers, scorers, and the log viewer</li>`,Hl,el,Fl,al,zl;return I=new bs({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),m=new j({props:{title:"Evaluating agents with Inspect AI",local:"evaluating-agents-with-inspect-ai",headingTag:"h1"}}),C=new j({props:{title:"How the pieces fit together",local:"how-the-pieces-fit-together",headingTag:"h2"}}),k=new d({props:{code:"VHJhaW4lMjB3aXRoJTIwT3BlbkVudiUyMChHUlBPJTIwJTJGJTIwU0ZUKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCVFMiU4NiU5MyUwQURlZmluZSUyMGFuJTIwSW5zcGVjdCUyMEFJJTIwVGFzayUwQSUyMCUyMC0lMjBkYXRhc2V0JTNBJTIwaGVsZC1vdXQlMjBlcGlzb2RlcyUyMG9yJTIwcHJvbXB0cyUwQSUyMCUyMC0lMjBzb2x2ZXIlM0ElMjBjYWxscyUyMHlvdXIlMjBtb2RlbCUyMCUyQiUyMHRoZSUyME9wZW5FbnYlMjBlbnYlMEElMjAlMjAtJTIwc2NvcmVyJTNBJTIwZ3JhZGVzJTIwY29ycmVjdG5lc3MlMjB1c2luZyUyMGVudiUyMHJld2FyZCUyMG9yJTIwZXhhY3QlMjBtYXRjaCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCVFMiU4NiU5MyUwQVJ1biUyMHZpYSUyMEluc3BlY3RBSUhhcm5lc3MlMjAlRTIlODYlOTIlMjBFdmFsUmVzdWx0JTIwd2l0aCUyMHN0cnVjdHVyZWQlMjBzY29yZXM=",highlighted:`Train <span class="hljs-built_in">with</span> OpenEnv (GRPO / SFT)
        ↓
Define an <span class="hljs-keyword">Inspect</span> AI Task
  - dataset: held-out episodes or prompts
  - solver: calls your model + the OpenEnv env
  - scorer: grades correctness <span class="hljs-built_in">using</span> env reward or <span class="hljs-built_in">exact</span> <span class="hljs-keyword">match</span>
        ↓
Run via InspectAIHarness → EvalResult <span class="hljs-built_in">with</span> structured scores`,lang:"",wrap:!1}}),Z=new j({props:{title:"Install dependencies",local:"install-dependencies",headingTag:"h2"}}),g=new d({props:{code:"cGlwJTIwaW5zdGFsbCUyMCUyMmluc3BlY3QtYWklM0UlM0QwLjMuMCUyMiUwQXBpcCUyMGluc3RhbGwlMjAlMjJvcGVuZW52JTIwJTQwJTIwZ2l0JTJCaHR0cHMlM0ElMkYlMkZnaXRodWIuY29tJTJGaHVnZ2luZ2ZhY2UlMkZPcGVuRW52LmdpdCUyMg==",highlighted:`pip install <span class="hljs-string">&quot;inspect-ai&gt;=0.3.0&quot;</span>
pip install <span class="hljs-string">&quot;openenv @ git+https://github.com/huggingface/OpenEnv.git&quot;</span>`,lang:"bash",wrap:!1}}),E=new j({props:{title:"Set your model provider",local:"set-your-model-provider",headingTag:"h2"}}),N=new d({props:{code:"aW1wb3J0JTIwZ2V0cGFzcyUyQyUyMG9zJTBBJTBBJTIzJTIwLS0tJTIwT3B0aW9uJTIwQSUzQSUyME9wZW5BSSUyMC0tLSUwQW9zLmVudmlyb24uc2V0ZGVmYXVsdCglMjJPUEVOQUlfQVBJX0tFWSUyMiUyQyUyMGdldHBhc3MuZ2V0cGFzcyglMjJPcGVuQUklMjBBUEklMjBrZXklM0ElMjAlMjIpKSUwQU1PREVMJTIwJTNEJTIwJTIyb3BlbmFpJTJGZ3B0LTUtbWluaSUyMiUwQSUwQSUyMyUyMC0tLSUyME9wdGlvbiUyMEIlM0ElMjBBbnRocm9waWMlMjAtLS0lMEElMjMlMjBvcy5lbnZpcm9uLnNldGRlZmF1bHQoJTIyQU5USFJPUElDX0FQSV9LRVklMjIlMkMlMjBnZXRwYXNzLmdldHBhc3MoJTIyQW50aHJvcGljJTIwQVBJJTIwa2V5JTNBJTIwJTIyKSklMEElMjMlMjBNT0RFTCUyMCUzRCUyMCUyMmFudGhyb3BpYyUyRmNsYXVkZS1oYWlrdS00LTUtMjAyNTEwMDElMjIlMEElMEElMjMlMjAtLS0lMjBPcHRpb24lMjBDJTNBJTIwbG9jYWwlMjB0cmFuc2Zvcm1lcnMlMjBtb2RlbCUyMChubyUyMEFQSSUyMGtleSUyMG5lZWRlZCklMjAtLS0lMEElMjMlMjBSZXF1aXJlcyUyMGElMjBHUFUlMjBmb3IlMjByZWFzb25hYmxlJTIwc3BlZWQuJTIwT21pdCUyMCd0ZW1wZXJhdHVyZSclMjBmcm9tJTIwZXZhbF9wYXJhbWV0ZXJzJTIwYmVsb3cuJTBBJTIzJTIwIXBpcCUyMGluc3RhbGwlMjAtVSUyMHRyYW5zZm9ybWVycyUwQSUyMyUyME1PREVMJTIwJTNEJTIwJTIyaGYlMkZRd2VuJTJGUXdlbjMuNS0wLjhCJTIyJTBBJTIzJTIwVXNlJTIwYSUyMGxvY2FsJTIwY2hlY2twb2ludCUyMHBhdGglMjB0byUyMHNraXAlMjB0aGUlMjBkb3dubG9hZCUzQSUwQSUyMyUyME1PREVMJTIwJTNEJTIwJTIyaGYlMkYuJTJGb3V0cHV0cyUyRm15LXRyYWluZWQtbW9kZWwlMjI=",highlighted:`<span class="hljs-keyword">import</span> getpass, os

<span class="hljs-comment"># --- Option A: OpenAI ---</span>
os.environ.setdefault(<span class="hljs-string">&quot;OPENAI_API_KEY&quot;</span>, getpass.getpass(<span class="hljs-string">&quot;OpenAI API key: &quot;</span>))
MODEL = <span class="hljs-string">&quot;openai/gpt-5-mini&quot;</span>

<span class="hljs-comment"># --- Option B: Anthropic ---</span>
<span class="hljs-comment"># os.environ.setdefault(&quot;ANTHROPIC_API_KEY&quot;, getpass.getpass(&quot;Anthropic API key: &quot;))</span>
<span class="hljs-comment"># MODEL = &quot;anthropic/claude-haiku-4-5-20251001&quot;</span>

<span class="hljs-comment"># --- Option C: local transformers model (no API key needed) ---</span>
<span class="hljs-comment"># Requires a GPU for reasonable speed. Omit &#x27;temperature&#x27; from eval_parameters below.</span>
<span class="hljs-comment"># !pip install -U transformers</span>
<span class="hljs-comment"># MODEL = &quot;hf/Qwen/Qwen3.5-0.8B&quot;</span>
<span class="hljs-comment"># Use a local checkpoint path to skip the download:</span>
<span class="hljs-comment"># MODEL = &quot;hf/./outputs/my-trained-model&quot;</span>`,lang:"python",wrap:!1}}),S=new j({props:{title:"Define an Inspect AI task for an OpenEnv environment",local:"define-an-inspect-ai-task-for-an-openenv-environment",headingTag:"h2"}}),Y=new d({props:{code:"aW1wb3J0JTIwYXN5bmNpbyUwQSUwQWZyb20lMjBpbnNwZWN0X2FpJTIwaW1wb3J0JTIwVGFzayUyQyUyMHRhc2slMEFmcm9tJTIwaW5zcGVjdF9haS5kYXRhc2V0JTIwaW1wb3J0JTIwU2FtcGxlJTBBZnJvbSUyMGluc3BlY3RfYWkuc2NvcmVyJTIwaW1wb3J0JTIwQ09SUkVDVCUyQyUyMElOQ09SUkVDVCUyQyUyMFNjb3JlJTJDJTIwVGFyZ2V0JTJDJTIwYWNjdXJhY3klMkMlMjBzY29yZXIlMEFmcm9tJTIwaW5zcGVjdF9haS5zb2x2ZXIlMjBpbXBvcnQlMjBHZW5lcmF0ZSUyQyUyMFRhc2tTdGF0ZSUyQyUyMHNvbHZlciUwQSUwQWZyb20lMjBvcGVuZW52LmNvcmUlMjBpbXBvcnQlMjBNQ1BUb29sQ2xpZW50JTBBJTBBRUNIT19FTlZfVVJMJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZvcGVuZW52LWVjaG8tZW52LmhmLnNwYWNlJTIyJTBBJTBBJTIzJTIwTGltaXQlMjBjb25jdXJyZW50JTIwZW52JTIwY29ubmVjdGlvbnMlMjB0byUyMG1hdGNoJTIwdGhlJTIwc2VydmVyJ3MlMjBNQVhfQ09OQ1VSUkVOVF9FTlZTLiUwQV9lbnZfc2VtJTIwJTNEJTIwYXN5bmNpby5TZW1hcGhvcmUoMSklMjAlMjAlMjMlMjBpbmNyZWFzZSUyMGlmJTIweW91ciUyMFNwYWNlJTIwc3VwcG9ydHMlMjBtb3JlJTIwc2Vzc2lvbnMlMEElMEElMEElNDB0YXNrJTBBZGVmJTIwb3BlbmVudl9lY2hvX2V2YWwoYmFzZV91cmwlM0ElMjBzdHIlMjAlM0QlMjBFQ0hPX0VOVl9VUkwpJTNBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwVGFzayglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkYXRhc2V0JTNEJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwU2FtcGxlKGlucHV0JTNEJTIyUmVwZWF0JTIwZXhhY3RseSUzQSUyMGhlbGxvJTIwd29ybGQlMjIlMkMlMjB0YXJnZXQlM0QlMjJoZWxsbyUyMHdvcmxkJTIyKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFNhbXBsZShpbnB1dCUzRCUyMlJlcGVhdCUyMGV4YWN0bHklM0ElMjBpbnNwZWN0JTIwYWklMjIlMkMlMjB0YXJnZXQlM0QlMjJpbnNwZWN0JTIwYWklMjIpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwU2FtcGxlKGlucHV0JTNEJTIyUmVwZWF0JTIwZXhhY3RseSUzQSUyMG9wZW5lbnYlMjBldmFsJTIyJTJDJTIwdGFyZ2V0JTNEJTIyb3BlbmVudiUyMGV2YWwlMjIpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwU2FtcGxlKGlucHV0JTNEJTIyUmVwZWF0JTIwZXhhY3RseSUzQSUyMHJlaW5mb3JjZW1lbnQlMjBsZWFybmluZyUyMiUyQyUyMHRhcmdldCUzRCUyMnJlaW5mb3JjZW1lbnQlMjBsZWFybmluZyUyMiklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBTYW1wbGUoaW5wdXQlM0QlMjJSZXBlYXQlMjBleGFjdGx5JTNBJTIwaHVnZ2luZyUyMGZhY2UlMjIlMkMlMjB0YXJnZXQlM0QlMjJodWdnaW5nJTIwZmFjZSUyMiklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzb2x2ZXIlM0RlY2hvX2Vudl9zb2x2ZXIoYmFzZV91cmwlM0RiYXNlX3VybCklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzY29yZXIlM0RlY2hvX3Njb3JlcigpJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUwQSU0MHNvbHZlciUwQWRlZiUyMGVjaG9fZW52X3NvbHZlcihiYXNlX3VybCUzQSUyMHN0ciklM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjJBc2slMjB0aGUlMjBtb2RlbCUyMHRvJTIwcmVwZWF0JTIwdGhlJTIwcGhyYXNlJTJDJTIwdGhlbiUyMGVjaG8lMjBpdCUyMHRocm91Z2glMjB0aGUlMjBlbnYuJTIyJTIyJTIyJTBBJTBBJTIwJTIwJTIwJTIwYXN5bmMlMjBkZWYlMjBzb2x2ZShzdGF0ZSUzQSUyMFRhc2tTdGF0ZSUyQyUyMGdlbmVyYXRlJTNBJTIwR2VuZXJhdGUpJTIwLSUzRSUyMFRhc2tTdGF0ZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXRlJTIwJTNEJTIwYXdhaXQlMjBnZW5lcmF0ZShzdGF0ZSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9vdXRwdXQlMjAlM0QlMjBzdGF0ZS5vdXRwdXQuY29tcGxldGlvbi5zdHJpcCgpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYXN5bmMlMjB3aXRoJTIwX2Vudl9zZW0lM0ElMjAlMjAlMjMlMjBvbmUlMjBlbnYlMjBjb25uZWN0aW9uJTIwYXQlMjBhJTIwdGltZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVudiUyMCUzRCUyME1DUFRvb2xDbGllbnQoYmFzZV91cmwlM0RiYXNlX3VybCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhd2FpdCUyMGVudi5yZXNldCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWNob2VkJTIwJTNEJTIwYXdhaXQlMjBlbnYuY2FsbF90b29sKCUyMmVjaG9fbWVzc2FnZSUyMiUyQyUyMG1lc3NhZ2UlM0Rtb2RlbF9vdXRwdXQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RhdGUubWV0YWRhdGElNUIlMjJlY2hvZWQlMjIlNUQlMjAlM0QlMjBzdHIoZWNob2VkKSUyMGlmJTIwZWNob2VkJTIwaXMlMjBub3QlMjBOb25lJTIwZWxzZSUyMCUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZpbmFsbHklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhd2FpdCUyMGVudi5jbG9zZSgpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc3RhdGUlMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBzb2x2ZSUwQSUwQSUwQSU0MHNjb3JlcihtZXRyaWNzJTNEJTVCYWNjdXJhY3koKSU1RCklMEFkZWYlMjBlY2hvX3Njb3JlcigpJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyQ09SUkVDVCUyMGlmJTIwdGhlJTIwZW52JTIwZWNob2VkJTIwYmFjayUyMGV4YWN0bHklMjB3aGF0JTIwdGhlJTIwdGFyZ2V0JTIwcGhyYXNlJTIwd2FzLiUyMiUyMiUyMiUwQSUwQSUyMCUyMCUyMCUyMGFzeW5jJTIwZGVmJTIwc2NvcmUoc3RhdGUlM0ElMjBUYXNrU3RhdGUlMkMlMjB0YXJnZXQlM0ElMjBUYXJnZXQpJTIwLSUzRSUyMFNjb3JlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWNob2VkJTIwJTNEJTIwc3RhdGUubWV0YWRhdGEuZ2V0KCUyMmVjaG9lZCUyMiUyQyUyMCUyMiUyMikuc3RyaXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV4cGVjdGVkJTIwJTNEJTIwdGFyZ2V0LnRleHQuc3RyaXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMFNjb3JlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHZhbHVlJTNEQ09SUkVDVCUyMGlmJTIwZWNob2VkJTIwJTNEJTNEJTIwZXhwZWN0ZWQlMjBlbHNlJTIwSU5DT1JSRUNUJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXhwbGFuYXRpb24lM0RmJTIyRW52JTIwZWNob2VkJTIwJTdCZWNob2VkIXIlN0QlMkMlMjBleHBlY3RlZCUyMCU3QmV4cGVjdGVkIXIlN0QlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2NvcmU=",highlighted:`<span class="hljs-keyword">import</span> asyncio

<span class="hljs-keyword">from</span> inspect_ai <span class="hljs-keyword">import</span> Task, task
<span class="hljs-keyword">from</span> inspect_ai.dataset <span class="hljs-keyword">import</span> Sample
<span class="hljs-keyword">from</span> inspect_ai.scorer <span class="hljs-keyword">import</span> CORRECT, INCORRECT, Score, Target, accuracy, scorer
<span class="hljs-keyword">from</span> inspect_ai.solver <span class="hljs-keyword">import</span> Generate, TaskState, solver

<span class="hljs-keyword">from</span> openenv.core <span class="hljs-keyword">import</span> MCPToolClient

ECHO_ENV_URL = <span class="hljs-string">&quot;https://openenv-echo-env.hf.space&quot;</span>

<span class="hljs-comment"># Limit concurrent env connections to match the server&#x27;s MAX_CONCURRENT_ENVS.</span>
_env_sem = asyncio.Semaphore(<span class="hljs-number">1</span>)  <span class="hljs-comment"># increase if your Space supports more sessions</span>


<span class="hljs-meta">@task</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">openenv_echo_eval</span>(<span class="hljs-params">base_url: <span class="hljs-built_in">str</span> = ECHO_ENV_URL</span>):
    <span class="hljs-keyword">return</span> Task(
        dataset=[
            Sample(<span class="hljs-built_in">input</span>=<span class="hljs-string">&quot;Repeat exactly: hello world&quot;</span>, target=<span class="hljs-string">&quot;hello world&quot;</span>),
            Sample(<span class="hljs-built_in">input</span>=<span class="hljs-string">&quot;Repeat exactly: inspect ai&quot;</span>, target=<span class="hljs-string">&quot;inspect ai&quot;</span>),
            Sample(<span class="hljs-built_in">input</span>=<span class="hljs-string">&quot;Repeat exactly: openenv eval&quot;</span>, target=<span class="hljs-string">&quot;openenv eval&quot;</span>),
            Sample(<span class="hljs-built_in">input</span>=<span class="hljs-string">&quot;Repeat exactly: reinforcement learning&quot;</span>, target=<span class="hljs-string">&quot;reinforcement learning&quot;</span>),
            Sample(<span class="hljs-built_in">input</span>=<span class="hljs-string">&quot;Repeat exactly: hugging face&quot;</span>, target=<span class="hljs-string">&quot;hugging face&quot;</span>),
        ],
        solver=echo_env_solver(base_url=base_url),
        scorer=echo_scorer(),
    )


<span class="hljs-meta">@solver</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">echo_env_solver</span>(<span class="hljs-params">base_url: <span class="hljs-built_in">str</span></span>):
    <span class="hljs-string">&quot;&quot;&quot;Ask the model to repeat the phrase, then echo it through the env.&quot;&quot;&quot;</span>

    <span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">solve</span>(<span class="hljs-params">state: TaskState, generate: Generate</span>) -&gt; TaskState:
        state = <span class="hljs-keyword">await</span> generate(state)
        model_output = state.output.completion.strip()

        <span class="hljs-keyword">async</span> <span class="hljs-keyword">with</span> _env_sem:  <span class="hljs-comment"># one env connection at a time</span>
            env = MCPToolClient(base_url=base_url)
            <span class="hljs-keyword">try</span>:
                <span class="hljs-keyword">await</span> env.reset()
                echoed = <span class="hljs-keyword">await</span> env.call_tool(<span class="hljs-string">&quot;echo_message&quot;</span>, message=model_output)
                state.metadata[<span class="hljs-string">&quot;echoed&quot;</span>] = <span class="hljs-built_in">str</span>(echoed) <span class="hljs-keyword">if</span> echoed <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;&quot;</span>
            <span class="hljs-keyword">finally</span>:
                <span class="hljs-keyword">await</span> env.close()

        <span class="hljs-keyword">return</span> state

    <span class="hljs-keyword">return</span> solve


<span class="hljs-meta">@scorer(<span class="hljs-params">metrics=[accuracy(<span class="hljs-params"></span>)]</span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">echo_scorer</span>():
    <span class="hljs-string">&quot;&quot;&quot;CORRECT if the env echoed back exactly what the target phrase was.&quot;&quot;&quot;</span>

    <span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">score</span>(<span class="hljs-params">state: TaskState, target: Target</span>) -&gt; Score:
        echoed = state.metadata.get(<span class="hljs-string">&quot;echoed&quot;</span>, <span class="hljs-string">&quot;&quot;</span>).strip()
        expected = target.text.strip()
        <span class="hljs-keyword">return</span> Score(
            value=CORRECT <span class="hljs-keyword">if</span> echoed == expected <span class="hljs-keyword">else</span> INCORRECT,
            explanation=<span class="hljs-string">f&quot;Env echoed <span class="hljs-subst">{echoed!r}</span>, expected <span class="hljs-subst">{expected!r}</span>&quot;</span>,
        )

    <span class="hljs-keyword">return</span> score`,lang:"python",wrap:!1}}),_=new j({props:{title:"Run the eval with InspectAIHarness",local:"run-the-eval-with-inspectaiharness",headingTag:"h2"}}),H=new d({props:{code:"aW1wb3J0JTIwaW5zcGVjdF9haSUwQWltcG9ydCUyMG9wZW5lbnYlMEElMEFmcm9tJTIwb3BlbmVudi5jb3JlLmV2YWxzJTIwaW1wb3J0JTIwRXZhbENvbmZpZyUyQyUyMEV2YWxSZXN1bHQlMkMlMjBJbnNwZWN0QUlIYXJuZXNzJTBBJTBBaGFybmVzcyUyMCUzRCUyMEluc3BlY3RBSUhhcm5lc3MobG9nX2RpciUzRCUyMi4lMkZldmFsLWxvZ3MlMjIpJTBBJTBBY29uZmlnJTIwJTNEJTIwRXZhbENvbmZpZyglMEElMjAlMjAlMjAlMjBoYXJuZXNzX25hbWUlM0QlMjJJbnNwZWN0QUlIYXJuZXNzJTIyJTJDJTBBJTIwJTIwJTIwJTIwaGFybmVzc192ZXJzaW9uJTNEaW5zcGVjdF9haS5fX3ZlcnNpb25fXyUyQyUwQSUyMCUyMCUyMCUyMGxpYnJhcnlfdmVyc2lvbnMlM0QlN0IlMjJvcGVuZW52JTIyJTNBJTIwb3BlbmVudi5fX3ZlcnNpb25fXyU3RCUyQyUwQSUyMCUyMCUyMCUyMGRhdGFzZXQlM0QlMjJvcGVuZW52X2VjaG9fZXZhbCUyMiUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfcGFyYW1ldGVycyUzRCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1vZGVsJTIyJTNBJTIwTU9ERUwlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0YXNrJTIyJTNBJTIwb3BlbmVudl9lY2hvX2V2YWwoYmFzZV91cmwlM0RFQ0hPX0VOVl9VUkwpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwdGVtcGVyYXR1cmUlMjBpcyUyMHN1cHBvcnRlZCUyMGZvciUyMEFQSSUyMHByb3ZpZGVycyUyMChPcHRpb25zJTIwQSUyRkIpLiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyME9taXQlMjBpdCUyMGZvciUyMGxvY2FsJTIwdHJhbnNmb3JtZXJzJTIwbW9kZWxzJTIwKE9wdGlvbiUyMEMpLiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRlbXBlcmF0dXJlJTIyJTNBJTIwMC4wJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBKSUwQSUwQXJlc3VsdCUzQSUyMEV2YWxSZXN1bHQlMjAlM0QlMjBoYXJuZXNzLnJ1bl9mcm9tX2NvbmZpZyhjb25maWcpJTBBcHJpbnQocmVzdWx0LnNjb3JlcyklMEElMjMlMjAlN0InYWNjdXJhY3knJTNBJTIwMS4wJTdE",highlighted:`<span class="hljs-keyword">import</span> inspect_ai
<span class="hljs-keyword">import</span> openenv

<span class="hljs-keyword">from</span> openenv.core.evals <span class="hljs-keyword">import</span> EvalConfig, EvalResult, InspectAIHarness

harness = InspectAIHarness(log_dir=<span class="hljs-string">&quot;./eval-logs&quot;</span>)

config = EvalConfig(
    harness_name=<span class="hljs-string">&quot;InspectAIHarness&quot;</span>,
    harness_version=inspect_ai.__version__,
    library_versions={<span class="hljs-string">&quot;openenv&quot;</span>: openenv.__version__},
    dataset=<span class="hljs-string">&quot;openenv_echo_eval&quot;</span>,
    eval_parameters={
        <span class="hljs-string">&quot;model&quot;</span>: MODEL,
        <span class="hljs-string">&quot;task&quot;</span>: openenv_echo_eval(base_url=ECHO_ENV_URL),
        <span class="hljs-comment"># temperature is supported for API providers (Options A/B).</span>
        <span class="hljs-comment"># Omit it for local transformers models (Option C).</span>
        <span class="hljs-string">&quot;temperature&quot;</span>: <span class="hljs-number">0.0</span>,
    },
)

result: EvalResult = harness.run_from_config(config)
<span class="hljs-built_in">print</span>(result.scores)
<span class="hljs-comment"># {&#x27;accuracy&#x27;: 1.0}</span>`,lang:"python",wrap:!1}}),z=new d({props:{code:"aW1wb3J0JTIwanNvbiUwQSUwQWNsYXNzJTIwX1N0ckZhbGxiYWNrKGpzb24uSlNPTkVuY29kZXIpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwZGVmYXVsdChzZWxmJTJDJTIwbyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBzdHIobyklMEElMEFwcmludChqc29uLmR1bXBzKHJlc3VsdC5tb2RlbF9kdW1wKCklMkMlMjBpbmRlbnQlM0QyJTJDJTIwY2xzJTNEX1N0ckZhbGxiYWNrKSk=",highlighted:`<span class="hljs-keyword">import</span> json

<span class="hljs-keyword">class</span> <span class="hljs-title class_">_StrFallback</span>(json.JSONEncoder):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">default</span>(<span class="hljs-params">self, o</span>):
        <span class="hljs-keyword">return</span> <span class="hljs-built_in">str</span>(o)

<span class="hljs-built_in">print</span>(json.dumps(result.model_dump(), indent=<span class="hljs-number">2</span>, cls=_StrFallback))`,lang:"python",wrap:!1}}),q=new j({props:{title:"Using a task file instead of a task object",local:"using-a-task-file-instead-of-a-task-object",headingTag:"h2"}}),L=new d({props:{code:"JTIzJTIwdGFza3MlMkZlY2hvX2V2YWwucHklMjAlMjAoY29udGFpbnMlMjB0aGUlMjAlNDB0YXNrJTIwZGVmaW5pdGlvbiUyMGFib3ZlKSUwQSUwQXJlc3VsdCUyMCUzRCUyMGhhcm5lc3MucnVuX2Zyb21fY29uZmlnKEV2YWxDb25maWcoJTBBJTIwJTIwJTIwJTIwaGFybmVzc19uYW1lJTNEJTIySW5zcGVjdEFJSGFybmVzcyUyMiUyQyUwQSUyMCUyMCUyMCUyMGhhcm5lc3NfdmVyc2lvbiUzRGluc3BlY3RfYWkuX192ZXJzaW9uX18lMkMlMEElMjAlMjAlMjAlMjBsaWJyYXJ5X3ZlcnNpb25zJTNEJTdCJTIyb3BlbmVudiUyMiUzQSUyMG9wZW5lbnYuX192ZXJzaW9uX18lN0QlMkMlMEElMjAlMjAlMjAlMjBkYXRhc2V0JTNEJTIydGFza3MlMkZlY2hvX2V2YWwucHklNDBvcGVuZW52X2VjaG9fZXZhbCUyMiUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfcGFyYW1ldGVycyUzRCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1vZGVsJTIyJTNBJTIwJTIyb3BlbmFpJTJGZ3B0LTUtbWluaSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRhc2slMjIlM0ElMjAlMjJ0YXNrcyUyRmVjaG9fZXZhbC5weSU0MG9wZW5lbnZfZWNob19ldmFsJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBKSk=",highlighted:`<span class="hljs-comment"># tasks/echo_eval.py  (contains the @task definition above)</span>

result = harness.run_from_config(EvalConfig(
    harness_name=<span class="hljs-string">&quot;InspectAIHarness&quot;</span>,
    harness_version=inspect_ai.__version__,
    library_versions={<span class="hljs-string">&quot;openenv&quot;</span>: openenv.__version__},
    dataset=<span class="hljs-string">&quot;tasks/echo_eval.py@openenv_echo_eval&quot;</span>,
    eval_parameters={
        <span class="hljs-string">&quot;model&quot;</span>: <span class="hljs-string">&quot;openai/gpt-5-mini&quot;</span>,
        <span class="hljs-string">&quot;task&quot;</span>: <span class="hljs-string">&quot;tasks/echo_eval.py@openenv_echo_eval&quot;</span>,
    },
))`,lang:"python",wrap:!1}}),O=new j({props:{title:"Adapting to your own environment and task",local:"adapting-to-your-own-environment-and-task",headingTag:"h2"}}),K=new d({props:{code:"aW1wb3J0JTIwYXN5bmNpbyUwQSUwQWZyb20lMjBpbnNwZWN0X2FpLnNvbHZlciUyMGltcG9ydCUyMEdlbmVyYXRlJTJDJTIwVGFza1N0YXRlJTJDJTIwc29sdmVyJTBBZnJvbSUyMG9wZW5lbnYuY29yZSUyMGltcG9ydCUyME1DUFRvb2xDbGllbnQlMEElMEFfZW52X3NlbSUyMCUzRCUyMGFzeW5jaW8uU2VtYXBob3JlKDEpJTIwJTIwJTIzJTIwcmFpc2UlMjBpZiUyMHlvdXIlMjBTcGFjZSUyMHN1cHBvcnRzJTIwbW9yZSUyMHNlc3Npb25zJTBBJTBBJTBBJTQwc29sdmVyJTBBZGVmJTIwbXlfZW52X3NvbHZlcihiYXNlX3VybCUzQSUyMHN0ciklM0ElMEElMjAlMjAlMjAlMjBhc3luYyUyMGRlZiUyMHNvbHZlKHN0YXRlJTNBJTIwVGFza1N0YXRlJTJDJTIwZ2VuZXJhdGUlM0ElMjBHZW5lcmF0ZSklMjAtJTNFJTIwVGFza1N0YXRlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RhdGUlMjAlM0QlMjBhd2FpdCUyMGdlbmVyYXRlKHN0YXRlKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsX291dHB1dCUyMCUzRCUyMHN0YXRlLm91dHB1dC5jb21wbGV0aW9uLnN0cmlwKCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhc3luYyUyMHdpdGglMjBfZW52X3NlbSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVudiUyMCUzRCUyME1DUFRvb2xDbGllbnQoYmFzZV91cmwlM0RiYXNlX3VybCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhd2FpdCUyMGVudi5yZXNldCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwYXdhaXQlMjBlbnYuY2FsbF90b29sKCUyMnlvdXJfdG9vbF9uYW1lJTIyJTJDJTIwbWVzc2FnZSUzRG1vZGVsX291dHB1dCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdGF0ZS5tZXRhZGF0YSU1QiUyMmVudl9yZXN1bHQlMjIlNUQlMjAlM0QlMjByZXN1bHQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmaW5hbGx5JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYXdhaXQlMjBlbnYuY2xvc2UoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMHN0YXRlJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwc29sdmU=",highlighted:`<span class="hljs-keyword">import</span> asyncio

<span class="hljs-keyword">from</span> inspect_ai.solver <span class="hljs-keyword">import</span> Generate, TaskState, solver
<span class="hljs-keyword">from</span> openenv.core <span class="hljs-keyword">import</span> MCPToolClient

_env_sem = asyncio.Semaphore(<span class="hljs-number">1</span>)  <span class="hljs-comment"># raise if your Space supports more sessions</span>


<span class="hljs-meta">@solver</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">my_env_solver</span>(<span class="hljs-params">base_url: <span class="hljs-built_in">str</span></span>):
    <span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">solve</span>(<span class="hljs-params">state: TaskState, generate: Generate</span>) -&gt; TaskState:
        state = <span class="hljs-keyword">await</span> generate(state)
        model_output = state.output.completion.strip()

        <span class="hljs-keyword">async</span> <span class="hljs-keyword">with</span> _env_sem:
            env = MCPToolClient(base_url=base_url)
            <span class="hljs-keyword">try</span>:
                <span class="hljs-keyword">await</span> env.reset()
                result = <span class="hljs-keyword">await</span> env.call_tool(<span class="hljs-string">&quot;your_tool_name&quot;</span>, message=model_output)
                state.metadata[<span class="hljs-string">&quot;env_result&quot;</span>] = result
            <span class="hljs-keyword">finally</span>:
                <span class="hljs-keyword">await</span> env.close()
        <span class="hljs-keyword">return</span> state

    <span class="hljs-keyword">return</span> solve`,lang:"python",wrap:!1}}),ll=new j({props:{title:"Next steps",local:"next-steps",headingTag:"h2"}}),el=new Cs({props:{source:"https://github.com/huggingface/openenv/blob/main/docs/source/tutorials/evaluation-inspect.md"}}),{c(){T=M("meta"),nl=a(),tl=M("p"),Ml=a(),i(I.$$.fragment),ol=a(),i(m.$$.fragment),pl=a(),u=M("p"),u.innerHTML=Ll,il=a(),b=M("p"),b.innerHTML=Ol,cl=a(),i(C.$$.fragment),rl=a(),v=M("p"),v.textContent=Pl,yl=a(),f=M("ul"),f.innerHTML=Dl,wl=a(),B=M("p"),B.innerHTML=Kl,Jl=a(),A=M("p"),A.textContent=ls,Tl=a(),i(k.$$.fragment),jl=a(),i(Z.$$.fragment),Ul=a(),i(g.$$.fragment),hl=a(),G=M("p"),G.innerHTML=ss,dl=a(),i(E.$$.fragment),Il=a(),V=M("p"),V.textContent=es,ml=a(),i(N.$$.fragment),ul=a(),X=M("p"),X.innerHTML=ts,bl=a(),i(S.$$.fragment),Cl=a(),Q=M("p"),Q.innerHTML=as,vl=a(),R=M("p"),R.innerHTML=ns,fl=a(),W=M("p"),W.innerHTML=Ms,Bl=a(),i(Y.$$.fragment),Al=a(),U=M("blockquote"),U.innerHTML=os,kl=a(),i(_.$$.fragment),Zl=a(),$=M("p"),$.innerHTML=ps,gl=a(),i(H.$$.fragment),Gl=a(),F=M("p"),F.innerHTML=is,El=a(),i(z.$$.fragment),Vl=a(),i(q.$$.fragment),Nl=a(),x=M("p"),x.innerHTML=cs,Xl=a(),i(L.$$.fragment),Sl=a(),i(O.$$.fragment),Ql=a(),P=M("p"),P.innerHTML=rs,Rl=a(),D=M("ol"),D.innerHTML=ys,Wl=a(),h=M("blockquote"),h.innerHTML=ws,Yl=a(),i(K.$$.fragment),_l=a(),i(ll.$$.fragment),$l=a(),sl=M("ul"),sl.innerHTML=Js,Hl=a(),i(el.$$.fragment),Fl=a(),al=M("p"),this.h()},l(l){const s=ms("svelte-u9bgzb",document.head);T=o(s,"META",{name:!0,content:!0}),s.forEach(e),nl=n(l),tl=o(l,"P",{}),Ts(tl).forEach(e),Ml=n(l),c(I.$$.fragment,l),ol=n(l),c(m.$$.fragment,l),pl=n(l),u=o(l,"P",{"data-svelte-h":!0}),p(u)!=="svelte-1cenuwu"&&(u.innerHTML=Ll),il=n(l),b=o(l,"P",{"data-svelte-h":!0}),p(b)!=="svelte-mr7257"&&(b.innerHTML=Ol),cl=n(l),c(C.$$.fragment,l),rl=n(l),v=o(l,"P",{"data-svelte-h":!0}),p(v)!=="svelte-in7fxo"&&(v.textContent=Pl),yl=n(l),f=o(l,"UL",{"data-svelte-h":!0}),p(f)!=="svelte-iezebx"&&(f.innerHTML=Dl),wl=n(l),B=o(l,"P",{"data-svelte-h":!0}),p(B)!=="svelte-js0uop"&&(B.innerHTML=Kl),Jl=n(l),A=o(l,"P",{"data-svelte-h":!0}),p(A)!=="svelte-1hejy7e"&&(A.textContent=ls),Tl=n(l),c(k.$$.fragment,l),jl=n(l),c(Z.$$.fragment,l),Ul=n(l),c(g.$$.fragment,l),hl=n(l),G=o(l,"P",{"data-svelte-h":!0}),p(G)!=="svelte-1wrnw4v"&&(G.innerHTML=ss),dl=n(l),c(E.$$.fragment,l),Il=n(l),V=o(l,"P",{"data-svelte-h":!0}),p(V)!=="svelte-v9d7i"&&(V.textContent=es),ml=n(l),c(N.$$.fragment,l),ul=n(l),X=o(l,"P",{"data-svelte-h":!0}),p(X)!=="svelte-1jm8jsi"&&(X.innerHTML=ts),bl=n(l),c(S.$$.fragment,l),Cl=n(l),Q=o(l,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-1kxwbnx"&&(Q.innerHTML=as),vl=n(l),R=o(l,"P",{"data-svelte-h":!0}),p(R)!=="svelte-1ov2w1s"&&(R.innerHTML=ns),fl=n(l),W=o(l,"P",{"data-svelte-h":!0}),p(W)!=="svelte-q0fjij"&&(W.innerHTML=Ms),Bl=n(l),c(Y.$$.fragment,l),Al=n(l),U=o(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),p(U)!=="svelte-o8j1kg"&&(U.innerHTML=os),kl=n(l),c(_.$$.fragment,l),Zl=n(l),$=o(l,"P",{"data-svelte-h":!0}),p($)!=="svelte-1mmc1or"&&($.innerHTML=ps),gl=n(l),c(H.$$.fragment,l),Gl=n(l),F=o(l,"P",{"data-svelte-h":!0}),p(F)!=="svelte-o254nq"&&(F.innerHTML=is),El=n(l),c(z.$$.fragment,l),Vl=n(l),c(q.$$.fragment,l),Nl=n(l),x=o(l,"P",{"data-svelte-h":!0}),p(x)!=="svelte-1d96re8"&&(x.innerHTML=cs),Xl=n(l),c(L.$$.fragment,l),Sl=n(l),c(O.$$.fragment,l),Ql=n(l),P=o(l,"P",{"data-svelte-h":!0}),p(P)!=="svelte-1epmotp"&&(P.innerHTML=rs),Rl=n(l),D=o(l,"OL",{"data-svelte-h":!0}),p(D)!=="svelte-1qj6khl"&&(D.innerHTML=ys),Wl=n(l),h=o(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),p(h)!=="svelte-n88mqq"&&(h.innerHTML=ws),Yl=n(l),c(K.$$.fragment,l),_l=n(l),c(ll.$$.fragment,l),$l=n(l),sl=o(l,"UL",{"data-svelte-h":!0}),p(sl)!=="svelte-1g6s95r"&&(sl.innerHTML=Js),Hl=n(l),c(el.$$.fragment,l),Fl=n(l),al=o(l,"P",{}),Ts(al).forEach(e),this.h()},h(){ql(T,"name","hf:doc:metadata"),ql(T,"content",fs),ql(U,"class","note"),ql(h,"class","tip")},m(l,s){us(document.head,T),t(l,nl,s),t(l,tl,s),t(l,Ml,s),r(I,l,s),t(l,ol,s),r(m,l,s),t(l,pl,s),t(l,u,s),t(l,il,s),t(l,b,s),t(l,cl,s),r(C,l,s),t(l,rl,s),t(l,v,s),t(l,yl,s),t(l,f,s),t(l,wl,s),t(l,B,s),t(l,Jl,s),t(l,A,s),t(l,Tl,s),r(k,l,s),t(l,jl,s),r(Z,l,s),t(l,Ul,s),r(g,l,s),t(l,hl,s),t(l,G,s),t(l,dl,s),r(E,l,s),t(l,Il,s),t(l,V,s),t(l,ml,s),r(N,l,s),t(l,ul,s),t(l,X,s),t(l,bl,s),r(S,l,s),t(l,Cl,s),t(l,Q,s),t(l,vl,s),t(l,R,s),t(l,fl,s),t(l,W,s),t(l,Bl,s),r(Y,l,s),t(l,Al,s),t(l,U,s),t(l,kl,s),r(_,l,s),t(l,Zl,s),t(l,$,s),t(l,gl,s),r(H,l,s),t(l,Gl,s),t(l,F,s),t(l,El,s),r(z,l,s),t(l,Vl,s),r(q,l,s),t(l,Nl,s),t(l,x,s),t(l,Xl,s),r(L,l,s),t(l,Sl,s),r(O,l,s),t(l,Ql,s),t(l,P,s),t(l,Rl,s),t(l,D,s),t(l,Wl,s),t(l,h,s),t(l,Yl,s),r(K,l,s),t(l,_l,s),r(ll,l,s),t(l,$l,s),t(l,sl,s),t(l,Hl,s),r(el,l,s),t(l,Fl,s),t(l,al,s),zl=!0},p:Us,i(l){zl||(y(I.$$.fragment,l),y(m.$$.fragment,l),y(C.$$.fragment,l),y(k.$$.fragment,l),y(Z.$$.fragment,l),y(g.$$.fragment,l),y(E.$$.fragment,l),y(N.$$.fragment,l),y(S.$$.fragment,l),y(Y.$$.fragment,l),y(_.$$.fragment,l),y(H.$$.fragment,l),y(z.$$.fragment,l),y(q.$$.fragment,l),y(L.$$.fragment,l),y(O.$$.fragment,l),y(K.$$.fragment,l),y(ll.$$.fragment,l),y(el.$$.fragment,l),zl=!0)},o(l){w(I.$$.fragment,l),w(m.$$.fragment,l),w(C.$$.fragment,l),w(k.$$.fragment,l),w(Z.$$.fragment,l),w(g.$$.fragment,l),w(E.$$.fragment,l),w(N.$$.fragment,l),w(S.$$.fragment,l),w(Y.$$.fragment,l),w(_.$$.fragment,l),w(H.$$.fragment,l),w(z.$$.fragment,l),w(q.$$.fragment,l),w(L.$$.fragment,l),w(O.$$.fragment,l),w(K.$$.fragment,l),w(ll.$$.fragment,l),w(el.$$.fragment,l),zl=!1},d(l){l&&(e(nl),e(tl),e(Ml),e(ol),e(pl),e(u),e(il),e(b),e(cl),e(rl),e(v),e(yl),e(f),e(wl),e(B),e(Jl),e(A),e(Tl),e(jl),e(Ul),e(hl),e(G),e(dl),e(Il),e(V),e(ml),e(ul),e(X),e(bl),e(Cl),e(Q),e(vl),e(R),e(fl),e(W),e(Bl),e(Al),e(U),e(kl),e(Zl),e($),e(gl),e(Gl),e(F),e(El),e(Vl),e(Nl),e(x),e(Xl),e(Sl),e(Ql),e(P),e(Rl),e(D),e(Wl),e(h),e(Yl),e(_l),e($l),e(sl),e(Hl),e(Fl),e(al)),e(T),J(I,l),J(m,l),J(C,l),J(k,l),J(Z,l),J(g,l),J(E,l),J(N,l),J(S,l),J(Y,l),J(_,l),J(H,l),J(z,l),J(q,l),J(L,l),J(O,l),J(K,l),J(ll,l),J(el,l)}}}const fs='{"title":"Evaluating agents with Inspect AI","local":"evaluating-agents-with-inspect-ai","sections":[{"title":"How the pieces fit together","local":"how-the-pieces-fit-together","sections":[],"depth":2},{"title":"Install dependencies","local":"install-dependencies","sections":[],"depth":2},{"title":"Set your model provider","local":"set-your-model-provider","sections":[],"depth":2},{"title":"Define an Inspect AI task for an OpenEnv environment","local":"define-an-inspect-ai-task-for-an-openenv-environment","sections":[],"depth":2},{"title":"Run the eval with InspectAIHarness","local":"run-the-eval-with-inspectaiharness","sections":[],"depth":2},{"title":"Using a task file instead of a task object","local":"using-a-task-file-instead-of-a-task-object","sections":[],"depth":2},{"title":"Adapting to your own environment and task","local":"adapting-to-your-own-environment-and-task","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';function Bs(xl){return hs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Es extends ds{constructor(T){super(),Is(this,T,Bs,vs,js,{})}}export{Es as component};