lighteval/pr_953/en/_app/immutable/nodes/3.bdc93b4e.js

import{s as wt,o as Jt,n as Tt}from"../chunks/scheduler.7da89386.js";import{S as Ut,i as jt,g as f,s as n,r as a,A as Ct,h as y,f as l,c as i,j as gt,u as m,x as g,k as dt,y as bt,a as s,v as p,d as r,t as c,w as o}from"../chunks/index.20910acc.js";import{T as ht}from"../chunks/Tip.53e22153.js";import{C as h}from"../chunks/CodeBlock.143bd81e.js";import{H as u,E as vt}from"../chunks/getInferenceSnippets.174c92b4.js";function It(ce){let M,d='To see an example of a custom metric added along with a custom task, look at the <a href="https://github.com/huggingface/lighteval/tree/main/examples/custom_tasks/ifeval" rel="nofollow">IFEval custom task</a>.';return{c(){M=f("p"),M.innerHTML=d},l($){M=y($,"P",{"data-svelte-h":!0}),g(M)!=="svelte-1418oma"&&(M.innerHTML=d)},m($,T){s($,M,T)},p:Tt,d($){$&&l(M)}}}function _t(ce){let M,d=`To contribute your custom metric to the Lighteval repository, you would first need
to install the required dev dependencies by running <code>pip install -e .[dev]</code>
and then run <code>pre-commit install</code> to install the pre-commit hooks.`;return{c(){M=f("p"),M.innerHTML=d},l($){M=y($,"P",{"data-svelte-h":!0}),g(M)!=="svelte-1ku0ebe"&&(M.innerHTML=d)},m($,T){s($,M,T)},p:Tt,d($){$&&l(M)}}}function kt(ce){let M,d,$,T,U,Me,j,ue,C,fe,b,lt="There are two types of metrics in Lighteval:",ye,v,$e,I,st="<li><strong>Purpose</strong>: Evaluate individual samples/predictions</li> <li><strong>Input</strong>: Takes a <code>Doc</code> and <code>ModelResponse</code> (model’s prediction)</li> <li><strong>Output</strong>: Returns a float or boolean value for that specific sample</li> <li><strong>Example</strong>: Checking if a model’s answer matches the correct answer for one sample</li>",ge,_,de,k,nt="<li><strong>Purpose</strong>: Compute final scores across the entire dataset/corpus</li> <li><strong>Input</strong>: Takes the results from all sample-level evaluations</li> <li><strong>Output</strong>: Returns a single score representing overall performance</li> <li><strong>Examples</strong>:<ul><li>Simple aggregation: Calculating average accuracy across all test samples</li> <li>Complex metrics: BLEU score where sample-level metric prepares data (tokenization, etc.) and corpus-level metric computes the actual BLEU score</li></ul></li>",he,G,Te,V,it=`First, check if you can use one of the parameterized functions in
<a href="package_reference/metrics#corpus-metrics">Corpus Metrics</a> or
<a href="package_reference/metrics#sample-metrics">Sample Metrics</a>.`,we,E,at="If not, you can use the <code>custom_task</code> system to register your new metric.",Je,w,Ue,J,je,Z,Ce,B,be,S,mt=`Create a new Python file which should contain the full logic of your metric.
The file also needs to start with these imports:`,ve,X,Ie,R,_e,x,pt=`You need to define a sample-level metric. All sample-level metrics will have the same signature, taking a
<code>~lighteval.types.Doc</code> and a <code>~lighteval.types.ModelResponse</code>. The metric should return a float or a
boolean.`,ke,Y,Ge,N,Ve,W,Ee,z,rt="If you want to return multiple metrics per sample, you need to return a dictionary with the metrics as keys and the values as values:",Ze,A,Be,Q,Se,L,ct="You can define an aggregation function if needed. A common aggregation function is <code>np.mean</code>:",Xe,H,Re,q,xe,F,Ye,P,ot=`If it’s a sample-level metric, you can use the following code
with <a href="/docs/lighteval/pr_953/en/package_reference/metrics#lighteval.metrics.utils.metric_utils.SampleLevelMetric">SampleLevelMetric</a>:`,Ne,D,We,O,ze,K,Mt=`If your metric defines multiple metrics per sample, you can use the following code
with <a href="/docs/lighteval/pr_953/en/package_reference/metrics#lighteval.metrics.utils.metric_utils.SampleLevelMetricGrouping">SampleLevelMetricGrouping</a>:`,Ae,ee,Qe,te,Le,le,ut=`To finish, add the following code so that it adds your metric to our metrics list
when loaded as a module:`,He,se,qe,ne,Fe,ie,Pe,ae,ft="You can then give your custom metric to Lighteval by using <code>--custom-tasks path_to_your_file</code> when launching it after adding it to the task config.",De,me,Oe,pe,Ke,re,et,oe,tt;return U=new u({props:{title:"Adding a New Metric",local:"adding-a-new-metric",headingTag:"h1"}}),j=new u({props:{title:"Before You Start",local:"before-you-start",headingTag:"h2"}}),C=new u({props:{title:"Two different types of metrics",local:"two-different-types-of-metrics",headingTag:"h3"}}),v=new u({props:{title:"Sample-Level Metrics",local:"sample-level-metrics",headingTag:"h4"}}),_=new u({props:{title:"Corpus-Level Metrics",local:"corpus-level-metrics",headingTag:"h4"}}),G=new u({props:{title:"Check Existing Metrics",local:"check-existing-metrics",headingTag:"h3"}}),w=new ht({props:{warning:!1,$$slots:{default:[It]},$$scope:{ctx:ce}}}),J=new ht({props:{warning:!0,$$slots:{default:[_t]},$$scope:{ctx:ce}}}),Z=new u({props:{title:"Creating a Custom Metric",local:"creating-a-custom-metric",headingTag:"h2"}}),B=new u({props:{title:"Step 1: Create the Metric File",local:"step-1-create-the-metric-file",headingTag:"h3"}}),X=new h({props:{code:"ZnJvbSUyMGFlbnVtJTIwaW1wb3J0JTIwZXh0ZW5kX2VudW0lMEFmcm9tJTIwbGlnaHRldmFsLm1ldHJpY3MlMjBpbXBvcnQlMjBNZXRyaWNz",highlighted:`<span class="hljs-keyword">from</span> aenum <span class="hljs-keyword">import</span> extend_enum
<span class="hljs-keyword">from</span> lighteval.metrics <span class="hljs-keyword">import</span> Metrics`,wrap:!1}}),R=new u({props:{title:"Step 2: Define the Sample-Level Metric",local:"step-2-define-the-sample-level-metric",headingTag:"h3"}}),Y=new u({props:{title:"Single Metric Example",local:"single-metric-example",headingTag:"h4"}}),N=new h({props:{code:"ZGVmJTIwY3VzdG9tX21ldHJpYyhkb2MlM0ElMjBEb2MlMkMlMjBtb2RlbF9yZXNwb25zZSUzQSUyME1vZGVsUmVzcG9uc2UpJTIwLSUzRSUyMGJvb2wlM0ElMEElMjAlMjAlMjAlMjByZXNwb25zZSUyMCUzRCUyMG1vZGVsX3Jlc3BvbnNlLnRleHQlNUIwJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwcmVzcG9uc2UlMjAlM0QlM0QlMjBkb2MuY2hvaWNlcyU1QmRvYy5nb2xkX2luZGV4JTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">custom_metric</span>(<span class="hljs-params">doc: Doc, model_response: ModelResponse</span>) -&gt; <span class="hljs-built_in">bool</span>:
    response = model_response.text[<span class="hljs-number">0</span>]
    <span class="hljs-keyword">return</span> response == doc.choices[doc.gold_index]`,wrap:!1}}),W=new u({props:{title:"Multiple Metrics Example",local:"multiple-metrics-example",headingTag:"h4"}}),A=new h({props:{code:"ZGVmJTIwY3VzdG9tX21ldHJpYyhkb2MlM0ElMjBEb2MlMkMlMjBtb2RlbF9yZXNwb25zZSUzQSUyME1vZGVsUmVzcG9uc2UpJTIwLSUzRSUyMGRpY3QlM0ElMEElMjAlMjAlMjAlMjByZXNwb25zZSUyMCUzRCUyMG1vZGVsX3Jlc3BvbnNlLnRleHQlNUIwJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTdCJTIyYWNjdXJhY3klMjIlM0ElMjByZXNwb25zZSUyMCUzRCUzRCUyMGRvYy5jaG9pY2VzJTVCZG9jLmdvbGRfaW5kZXglNUQlMkMlMjAlMjJvdGhlcl9tZXRyaWMlMjIlM0ElMjAwLjUlN0Q=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">custom_metric</span>(<span class="hljs-params">doc: Doc, model_response: ModelResponse</span>) -&gt; <span class="hljs-built_in">dict</span>:
    response = model_response.text[<span class="hljs-number">0</span>]
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;accuracy&quot;</span>: response == doc.choices[doc.gold_index], <span class="hljs-string">&quot;other_metric&quot;</span>: <span class="hljs-number">0.5</span>}`,wrap:!1}}),Q=new u({props:{title:"Step 3: Define Aggregation Function (Optional)",local:"step-3-define-aggregation-function-optional",headingTag:"h3"}}),H=new h({props:{code:"ZGVmJTIwYWdnX2Z1bmN0aW9uKGl0ZW1zKSUzQSUwQSUyMCUyMCUyMCUyMGZsYXRfaXRlbXMlMjAlM0QlMjAlNUJpdGVtJTIwZm9yJTIwc3VibGlzdCUyMGluJTIwaXRlbXMlMjBmb3IlMjBpdGVtJTIwaW4lMjBzdWJsaXN0JTVEJTBBJTIwJTIwJTIwJTIwc2NvcmUlMjAlM0QlMjBzdW0oZmxhdF9pdGVtcyklMjAlMkYlMjBsZW4oZmxhdF9pdGVtcyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBzY29yZQ==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">agg_function</span>(<span class="hljs-params">items</span>):
    flat_items = [item <span class="hljs-keyword">for</span> sublist <span class="hljs-keyword">in</span> items <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> sublist]
    score = <span class="hljs-built_in">sum</span>(flat_items) / <span class="hljs-built_in">len</span>(flat_items)
    <span class="hljs-keyword">return</span> score`,wrap:!1}}),q=new u({props:{title:"Step 4: Create the Metric Object",local:"step-4-create-the-metric-object",headingTag:"h3"}}),F=new u({props:{title:"Single Metric",local:"single-metric",headingTag:"h4"}}),D=new h({props:{code:"bXlfY3VzdG9tX21ldHJpYyUyMCUzRCUyMFNhbXBsZUxldmVsTWV0cmljKCUwQSUyMCUyMCUyMCUyMG1ldHJpY19uYW1lJTNEJTIyY3VzdG9tX2FjY3VyYWN5JTIyJTJDJTBBJTIwJTIwJTIwJTIwaGlnaGVyX2lzX2JldHRlciUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBjYXRlZ29yeSUzRFNhbXBsaW5nTWV0aG9kLkdFTkVSQVRJVkUlMkMlMEElMjAlMjAlMjAlMjBzYW1wbGVfbGV2ZWxfZm4lM0RjdXN0b21fbWV0cmljJTJDJTBBJTIwJTIwJTIwJTIwY29ycHVzX2xldmVsX2ZuJTNEYWdnX2Z1bmN0aW9uJTJDJTBBKQ==",highlighted:`my_custom_metric = SampleLevelMetric(
    metric_name=<span class="hljs-string">&quot;custom_accuracy&quot;</span>,
    higher_is_better=<span class="hljs-literal">True</span>,
    category=SamplingMethod.GENERATIVE,
    sample_level_fn=custom_metric,
    corpus_level_fn=agg_function,
)`,wrap:!1}}),O=new u({props:{title:"Multiple Metrics",local:"multiple-metrics",headingTag:"h4"}}),ee=new h({props:{code:"Y3VzdG9tX21ldHJpYyUyMCUzRCUyMFNhbXBsZUxldmVsTWV0cmljR3JvdXBpbmcoJTBBJTIwJTIwJTIwJTIwbWV0cmljX25hbWUlM0QlNUIlMjJhY2N1cmFjeSUyMiUyQyUyMCUyMnJlc3BvbnNlX2xlbmd0aCUyMiUyQyUyMCUyMmNvbmZpZGVuY2UlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBoaWdoZXJfaXNfYmV0dGVyJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWNjdXJhY3klMjIlM0ElMjBUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycmVzcG9uc2VfbGVuZ3RoJTIyJTNBJTIwRmFsc2UlMkMlMjAlMjAlMjMlMjBTaG9ydGVyJTIwcmVzcG9uc2VzJTIwbWlnaHQlMjBiZSUyMGJldHRlciUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbmZpZGVuY2UlMjIlM0ElMjBUcnVlJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwY2F0ZWdvcnklM0RTYW1wbGluZ01ldGhvZC5HRU5FUkFUSVZFJTJDJTBBJTIwJTIwJTIwJTIwc2FtcGxlX2xldmVsX2ZuJTNEY3VzdG9tX21ldHJpYyUyQyUwQSUyMCUyMCUyMCUyMGNvcnB1c19sZXZlbF9mbiUzRCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmFjY3VyYWN5JTIyJTNBJTIwbnAubWVhbiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJlc3BvbnNlX2xlbmd0aCUyMiUzQSUyMG5wLm1lYW4lMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJjb25maWRlbmNlJTIyJTNBJTIwbnAubWVhbiUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSk=",highlighted:`custom_metric = SampleLevelMetricGrouping(
    metric_name=[<span class="hljs-string">&quot;accuracy&quot;</span>, <span class="hljs-string">&quot;response_length&quot;</span>, <span class="hljs-string">&quot;confidence&quot;</span>],
    higher_is_better={
        <span class="hljs-string">&quot;accuracy&quot;</span>: <span class="hljs-literal">True</span>,
        <span class="hljs-string">&quot;response_length&quot;</span>: <span class="hljs-literal">False</span>,  <span class="hljs-comment"># Shorter responses might be better</span>
        <span class="hljs-string">&quot;confidence&quot;</span>: <span class="hljs-literal">True</span>
    },
    category=SamplingMethod.GENERATIVE,
    sample_level_fn=custom_metric,
    corpus_level_fn={
        <span class="hljs-string">&quot;accuracy&quot;</span>: np.mean,
        <span class="hljs-string">&quot;response_length&quot;</span>: np.mean,
        <span class="hljs-string">&quot;confidence&quot;</span>: np.mean,
    },
)`,wrap:!1}}),te=new u({props:{title:"Step 5: Register the Metric",local:"step-5-register-the-metric",headingTag:"h3"}}),se=new h({props:{code:"JTIzJTIwQWRkcyUyMHRoZSUyMG1ldHJpYyUyMHRvJTIwdGhlJTIwbWV0cmljJTIwbGlzdCElMEFleHRlbmRfZW51bShNZXRyaWNzJTJDJTIwJTIyQ1VTVE9NX0FDQ1VSQUNZJTIyJTJDJTIwbXlfY3VzdG9tX21ldHJpYyklMEElMEFpZiUyMF9fbmFtZV9fJTIwJTNEJTNEJTIwJTIyX19tYWluX18lMjIlM0ElMEElMjAlMjAlMjAlMjBwcmludCglMjJJbXBvcnRlZCUyMG1ldHJpYyUyMik=",highlighted:`<span class="hljs-comment"># Adds the metric to the metric list!</span>
extend_enum(Metrics, <span class="hljs-string">&quot;CUSTOM_ACCURACY&quot;</span>, my_custom_metric)

<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Imported metric&quot;</span>)`,wrap:!1}}),ne=new u({props:{title:"Using Your Custom Metric",local:"using-your-custom-metric",headingTag:"h2"}}),ie=new u({props:{title:"With Custom Tasks",local:"with-custom-tasks",headingTag:"h3"}}),me=new h({props:{code:"bGlnaHRldmFsJTIwYWNjZWxlcmF0ZSUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMm1vZGVsX25hbWUlM0RvcGVuYWktY29tbXVuaXR5JTJGZ3B0MiUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMmxlYWRlcmJvYXJkJTdDdHJ1dGhmdWxxYSUzQW1jJTdDMCUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tY3VzdG9tLXRhc2tzJTIwcGF0aF90b195b3VyX21ldHJpY19maWxlLnB5",highlighted:`lighteval accelerate \\
    <span class="hljs-string">&quot;model_name=openai-community/gpt2&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span> \\
    --custom-tasks path_to_your_metric_file.py`,wrap:!1}}),pe=new h({props:{code:"ZnJvbSUyMGxpZ2h0ZXZhbC50YXNrcy5saWdodGV2YWxfdGFzayUyMGltcG9ydCUyMExpZ2h0ZXZhbFRhc2tDb25maWclMEElMEF0YXNrJTIwJTNEJTIwTGlnaHRldmFsVGFza0NvbmZpZyglMEElMjAlMjAlMjAlMjBuYW1lJTNEJTIybXlfY3VzdG9tX3Rhc2slMjIlMkMlMEElMjAlMjAlMjAlMjBzdWl0ZSUzRCU1QiUyMmNvbW11bml0eSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMG1ldHJpYyUzRCU1Qm15X2N1c3RvbV9tZXRyaWMlNUQlMkMlMjAlMjAlMjMlMjBVc2UlMjB5b3VyJTIwY3VzdG9tJTIwbWV0cmljJTIwaGVyZSUwQSUyMCUyMCUyMCUyMHByb21wdF9mdW5jdGlvbiUzRG15X3Byb21wdF9mdW5jdGlvbiUyQyUwQSUyMCUyMCUyMCUyMGhmX3JlcG8lM0QlMjJteV9kYXRhc2V0JTIyJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbl9zcGxpdHMlM0QlNUIlMjJ0ZXN0JTIyJTVEJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> lighteval.tasks.lighteval_task <span class="hljs-keyword">import</span> LightevalTaskConfig

task = LightevalTaskConfig(
    name=<span class="hljs-string">&quot;my_custom_task&quot;</span>,
    suite=[<span class="hljs-string">&quot;community&quot;</span>],
    metric=[my_custom_metric],  <span class="hljs-comment"># Use your custom metric here</span>
    prompt_function=my_prompt_function,
    hf_repo=<span class="hljs-string">&quot;my_dataset&quot;</span>,
    evaluation_splits=[<span class="hljs-string">&quot;test&quot;</span>]
)`,wrap:!1}}),re=new vt({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/adding-a-new-metric.mdx"}}),{c(){M=f("meta"),d=n(),$=f("p"),T=n(),a(U.$$.fragment),Me=n(),a(j.$$.fragment),ue=n(),a(C.$$.fragment),fe=n(),b=f("p"),b.textContent=lt,ye=n(),a(v.$$.fragment),$e=n(),I=f("ul"),I.innerHTML=st,ge=n(),a(_.$$.fragment),de=n(),k=f("ul"),k.innerHTML=nt,he=n(),a(G.$$.fragment),Te=n(),V=f("p"),V.innerHTML=it,we=n(),E=f("p"),E.innerHTML=at,Je=n(),a(w.$$.fragment),Ue=n(),a(J.$$.fragment),je=n(),a(Z.$$.fragment),Ce=n(),a(B.$$.fragment),be=n(),S=f("p"),S.textContent=mt,ve=n(),a(X.$$.fragment),Ie=n(),a(R.$$.fragment),_e=n(),x=f("p"),x.innerHTML=pt,ke=n(),a(Y.$$.fragment),Ge=n(),a(N.$$.fragment),Ve=n(),a(W.$$.fragment),Ee=n(),z=f("p"),z.textContent=rt,Ze=n(),a(A.$$.fragment),Be=n(),a(Q.$$.fragment),Se=n(),L=f("p"),L.innerHTML=ct,Xe=n(),a(H.$$.fragment),Re=n(),a(q.$$.fragment),xe=n(),a(F.$$.fragment),Ye=n(),P=f("p"),P.innerHTML=ot,Ne=n(),a(D.$$.fragment),We=n(),a(O.$$.fragment),ze=n(),K=f("p"),K.innerHTML=Mt,Ae=n(),a(ee.$$.fragment),Qe=n(),a(te.$$.fragment),Le=n(),le=f("p"),le.textContent=ut,He=n(),a(se.$$.fragment),qe=n(),a(ne.$$.fragment),Fe=n(),a(ie.$$.fragment),Pe=n(),ae=f("p"),ae.innerHTML=ft,De=n(),a(me.$$.fragment),Oe=n(),a(pe.$$.fragment),Ke=n(),a(re.$$.fragment),et=n(),oe=f("p"),this.h()},l(e){const t=Ct("svelte-u9bgzb",document.head);M=y(t,"META",{name:!0,content:!0}),t.forEach(l),d=i(e),$=y(e,"P",{}),gt($).forEach(l),T=i(e),m(U.$$.fragment,e),Me=i(e),m(j.$$.fragment,e),ue=i(e),m(C.$$.fragment,e),fe=i(e),b=y(e,"P",{"data-svelte-h":!0}),g(b)!=="svelte-1jnixsg"&&(b.textContent=lt),ye=i(e),m(v.$$.fragment,e),$e=i(e),I=y(e,"UL",{"data-svelte-h":!0}),g(I)!=="svelte-1nwago0"&&(I.innerHTML=st),ge=i(e),m(_.$$.fragment,e),de=i(e),k=y(e,"UL",{"data-svelte-h":!0}),g(k)!=="svelte-1b4pgyk"&&(k.innerHTML=nt),he=i(e),m(G.$$.fragment,e),Te=i(e),V=y(e,"P",{"data-svelte-h":!0}),g(V)!=="svelte-vpq2qz"&&(V.innerHTML=it),we=i(e),E=y(e,"P",{"data-svelte-h":!0}),g(E)!=="svelte-xdscpl"&&(E.innerHTML=at),Je=i(e),m(w.$$.fragment,e),Ue=i(e),m(J.$$.fragment,e),je=i(e),m(Z.$$.fragment,e),Ce=i(e),m(B.$$.fragment,e),be=i(e),S=y(e,"P",{"data-svelte-h":!0}),g(S)!=="svelte-rk8zi4"&&(S.textContent=mt),ve=i(e),m(X.$$.fragment,e),Ie=i(e),m(R.$$.fragment,e),_e=i(e),x=y(e,"P",{"data-svelte-h":!0}),g(x)!=="svelte-uw89ed"&&(x.innerHTML=pt),ke=i(e),m(Y.$$.fragment,e),Ge=i(e),m(N.$$.fragment,e),Ve=i(e),m(W.$$.fragment,e),Ee=i(e),z=y(e,"P",{"data-svelte-h":!0}),g(z)!=="svelte-1vuyefu"&&(z.textContent=rt),Ze=i(e),m(A.$$.fragment,e),Be=i(e),m(Q.$$.fragment,e),Se=i(e),L=y(e,"P",{"data-svelte-h":!0}),g(L)!=="svelte-n3kro3"&&(L.innerHTML=ct),Xe=i(e),m(H.$$.fragment,e),Re=i(e),m(q.$$.fragment,e),xe=i(e),m(F.$$.fragment,e),Ye=i(e),P=y(e,"P",{"data-svelte-h":!0}),g(P)!=="svelte-vbs0k8"&&(P.innerHTML=ot),Ne=i(e),m(D.$$.fragment,e),We=i(e),m(O.$$.fragment,e),ze=i(e),K=y(e,"P",{"data-svelte-h":!0}),g(K)!=="svelte-1aoa6e6"&&(K.innerHTML=Mt),Ae=i(e),m(ee.$$.fragment,e),Qe=i(e),m(te.$$.fragment,e),Le=i(e),le=y(e,"P",{"data-svelte-h":!0}),g(le)!=="svelte-1bbv5uf"&&(le.textContent=ut),He=i(e),m(se.$$.fragment,e),qe=i(e),m(ne.$$.fragment,e),Fe=i(e),m(ie.$$.fragment,e),Pe=i(e),ae=y(e,"P",{"data-svelte-h":!0}),g(ae)!=="svelte-o8irch"&&(ae.innerHTML=ft),De=i(e),m(me.$$.fragment,e),Oe=i(e),m(pe.$$.fragment,e),Ke=i(e),m(re.$$.fragment,e),et=i(e),oe=y(e,"P",{}),gt(oe).forEach(l),this.h()},h(){dt(M,"name","hf:doc:metadata"),dt(M,"content",Gt)},m(e,t){bt(document.head,M),s(e,d,t),s(e,$,t),s(e,T,t),p(U,e,t),s(e,Me,t),p(j,e,t),s(e,ue,t),p(C,e,t),s(e,fe,t),s(e,b,t),s(e,ye,t),p(v,e,t),s(e,$e,t),s(e,I,t),s(e,ge,t),p(_,e,t),s(e,de,t),s(e,k,t),s(e,he,t),p(G,e,t),s(e,Te,t),s(e,V,t),s(e,we,t),s(e,E,t),s(e,Je,t),p(w,e,t),s(e,Ue,t),p(J,e,t),s(e,je,t),p(Z,e,t),s(e,Ce,t),p(B,e,t),s(e,be,t),s(e,S,t),s(e,ve,t),p(X,e,t),s(e,Ie,t),p(R,e,t),s(e,_e,t),s(e,x,t),s(e,ke,t),p(Y,e,t),s(e,Ge,t),p(N,e,t),s(e,Ve,t),p(W,e,t),s(e,Ee,t),s(e,z,t),s(e,Ze,t),p(A,e,t),s(e,Be,t),p(Q,e,t),s(e,Se,t),s(e,L,t),s(e,Xe,t),p(H,e,t),s(e,Re,t),p(q,e,t),s(e,xe,t),p(F,e,t),s(e,Ye,t),s(e,P,t),s(e,Ne,t),p(D,e,t),s(e,We,t),p(O,e,t),s(e,ze,t),s(e,K,t),s(e,Ae,t),p(ee,e,t),s(e,Qe,t),p(te,e,t),s(e,Le,t),s(e,le,t),s(e,He,t),p(se,e,t),s(e,qe,t),p(ne,e,t),s(e,Fe,t),p(ie,e,t),s(e,Pe,t),s(e,ae,t),s(e,De,t),p(me,e,t),s(e,Oe,t),p(pe,e,t),s(e,Ke,t),p(re,e,t),s(e,et,t),s(e,oe,t),tt=!0},p(e,[t]){const yt={};t&2&&(yt.$$scope={dirty:t,ctx:e}),w.$set(yt);const $t={};t&2&&($t.$$scope={dirty:t,ctx:e}),J.$set($t)},i(e){tt||(r(U.$$.fragment,e),r(j.$$.fragment,e),r(C.$$.fragment,e),r(v.$$.fragment,e),r(_.$$.fragment,e),r(G.$$.fragment,e),r(w.$$.fragment,e),r(J.$$.fragment,e),r(Z.$$.fragment,e),r(B.$$.fragment,e),r(X.$$.fragment,e),r(R.$$.fragment,e),r(Y.$$.fragment,e),r(N.$$.fragment,e),r(W.$$.fragment,e),r(A.$$.fragment,e),r(Q.$$.fragment,e),r(H.$$.fragment,e),r(q.$$.fragment,e),r(F.$$.fragment,e),r(D.$$.fragment,e),r(O.$$.fragment,e),r(ee.$$.fragment,e),r(te.$$.fragment,e),r(se.$$.fragment,e),r(ne.$$.fragment,e),r(ie.$$.fragment,e),r(me.$$.fragment,e),r(pe.$$.fragment,e),r(re.$$.fragment,e),tt=!0)},o(e){c(U.$$.fragment,e),c(j.$$.fragment,e),c(C.$$.fragment,e),c(v.$$.fragment,e),c(_.$$.fragment,e),c(G.$$.fragment,e),c(w.$$.fragment,e),c(J.$$.fragment,e),c(Z.$$.fragment,e),c(B.$$.fragment,e),c(X.$$.fragment,e),c(R.$$.fragment,e),c(Y.$$.fragment,e),c(N.$$.fragment,e),c(W.$$.fragment,e),c(A.$$.fragment,e),c(Q.$$.fragment,e),c(H.$$.fragment,e),c(q.$$.fragment,e),c(F.$$.fragment,e),c(D.$$.fragment,e),c(O.$$.fragment,e),c(ee.$$.fragment,e),c(te.$$.fragment,e),c(se.$$.fragment,e),c(ne.$$.fragment,e),c(ie.$$.fragment,e),c(me.$$.fragment,e),c(pe.$$.fragment,e),c(re.$$.fragment,e),tt=!1},d(e){e&&(l(d),l($),l(T),l(Me),l(ue),l(fe),l(b),l(ye),l($e),l(I),l(ge),l(de),l(k),l(he),l(Te),l(V),l(we),l(E),l(Je),l(Ue),l(je),l(Ce),l(be),l(S),l(ve),l(Ie),l(_e),l(x),l(ke),l(Ge),l(Ve),l(Ee),l(z),l(Ze),l(Be),l(Se),l(L),l(Xe),l(Re),l(xe),l(Ye),l(P),l(Ne),l(We),l(ze),l(K),l(Ae),l(Qe),l(Le),l(le),l(He),l(qe),l(Fe),l(Pe),l(ae),l(De),l(Oe),l(Ke),l(et),l(oe)),l(M),o(U,e),o(j,e),o(C,e),o(v,e),o(_,e),o(G,e),o(w,e),o(J,e),o(Z,e),o(B,e),o(X,e),o(R,e),o(Y,e),o(N,e),o(W,e),o(A,e),o(Q,e),o(H,e),o(q,e),o(F,e),o(D,e),o(O,e),o(ee,e),o(te,e),o(se,e),o(ne,e),o(ie,e),o(me,e),o(pe,e),o(re,e)}}}const Gt='{"title":"Adding a New Metric","local":"adding-a-new-metric","sections":[{"title":"Before You Start","local":"before-you-start","sections":[{"title":"Two different types of metrics","local":"two-different-types-of-metrics","sections":[{"title":"Sample-Level Metrics","local":"sample-level-metrics","sections":[],"depth":4},{"title":"Corpus-Level Metrics","local":"corpus-level-metrics","sections":[],"depth":4}],"depth":3},{"title":"Check Existing Metrics","local":"check-existing-metrics","sections":[],"depth":3}],"depth":2},{"title":"Creating a Custom Metric","local":"creating-a-custom-metric","sections":[{"title":"Step 1: Create the Metric File","local":"step-1-create-the-metric-file","sections":[],"depth":3},{"title":"Step 2: Define the Sample-Level Metric","local":"step-2-define-the-sample-level-metric","sections":[{"title":"Single Metric Example","local":"single-metric-example","sections":[],"depth":4},{"title":"Multiple Metrics Example","local":"multiple-metrics-example","sections":[],"depth":4}],"depth":3},{"title":"Step 3: Define Aggregation Function (Optional)","local":"step-3-define-aggregation-function-optional","sections":[],"depth":3},{"title":"Step 4: Create the Metric Object","local":"step-4-create-the-metric-object","sections":[{"title":"Single Metric","local":"single-metric","sections":[],"depth":4},{"title":"Multiple Metrics","local":"multiple-metrics","sections":[],"depth":4}],"depth":3},{"title":"Step 5: Register the Metric","local":"step-5-register-the-metric","sections":[],"depth":3}],"depth":2},{"title":"Using Your Custom Metric","local":"using-your-custom-metric","sections":[{"title":"With Custom Tasks","local":"with-custom-tasks","sections":[],"depth":3}],"depth":2}],"depth":1}';function Vt(ce){return Jt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rt extends Ut{constructor(M){super(),jt(this,M,Vt,kt,wt,{})}}export{Rt as component};