lighteval/pr_1264/en/_app/immutable/nodes/3.7bb8babd.js

import{s as wt,n as Jt,o as Ut}from"../chunks/scheduler.3a17fb72.js";import{S as jt,i as Ct,e as M,s as n,c as a,h as bt,a as u,d as l,b as i,f as dt,g as p,j as y,k as st,l as vt,m as s,n as m,t as r,o as c,p as o}from"../chunks/index.093f8863.js";import{C as It,H as f,E as _t}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.1a6c7b74.js";import{C as h}from"../chunks/CodeBlock.321b9d9c.js";function kt(nt){let g,ce,me,oe,d,Me,w,ue,J,fe,U,ye,j,it="There are two types of metrics in Lighteval:",ge,C,he,b,at="<li><strong>Purpose</strong>: Evaluate individual samples/predictions</li> <li><strong>Input</strong>: Takes a <code>Doc</code> and <code>ModelResponse</code> (model’s prediction)</li> <li><strong>Output</strong>: Returns a float or boolean value for that specific sample</li> <li><strong>Example</strong>: Checking if a model’s answer matches the correct answer for one sample</li>",$e,v,Te,I,pt=`<li><strong>Purpose</strong>: Compute final scores across the entire dataset/corpus</li> <li><strong>Input</strong>: Takes the results from all sample-level evaluations</li> <li><strong>Output</strong>: Returns a single score representing overall performance</li> <li><strong>Examples</strong>:
<ul><li>Simple aggregation: Calculating average accuracy across all test samples</li> <li>Complex metrics: BLEU score where sample-level metric prepares data (tokenization, etc.) and corpus-level metric computes the actual BLEU score</li></ul></li>`,de,_,we,k,mt=`First, check if you can use one of the parameterized functions in
<a href="package_reference/metrics#corpus-metrics">Corpus Metrics</a> or
<a href="package_reference/metrics#sample-metrics">Sample Metrics</a>.`,Je,V,rt="If not, you can use the <code>custom_task</code> system to register your new metric.",Ue,$,ct='<p>To see an example of a custom metric added along with a custom task, look at the <a href="https://github.com/huggingface/lighteval/tree/main/examples/custom_tasks/ifeval" rel="nofollow">IFEval custom task</a>.</p>',je,T,ot=`<p>To contribute your custom metric to the Lighteval repository, you would first need
to install the required dev dependencies by running <code>pip install -e .[dev]</code>
and then run <code>pre-commit install</code> to install the pre-commit hooks.</p>`,Ce,E,be,G,ve,Z,Mt=`Create a new Python file which should contain the full logic of your metric.
The file also needs to start with these imports:`,Ie,X,_e,B,ke,R,ut=`You need to define a sample-level metric. All sample-level metrics will have the same signature, taking a
<code>~lighteval.types.Doc</code> and a <code>~lighteval.types.ModelResponse</code>. The metric should return a float or a
boolean.`,Ve,x,Ee,S,Ge,Y,Ze,N,ft="If you want to return multiple metrics per sample, you need to return a dictionary with the metrics as keys and the values as values:",Xe,A,Be,z,Re,W,yt="You can define an aggregation function if needed. A common aggregation function is <code>np.mean</code>:",xe,L,Se,Q,Ye,H,Ne,q,gt=`If it’s a sample-level metric, you can use the following code
with <a href="/docs/lighteval/pr_1264/en/package_reference/metrics#lighteval.metrics.utils.metric_utils.SampleLevelMetric">SampleLevelMetric</a>:`,Ae,F,ze,P,We,D,ht=`If your metric defines multiple metrics per sample, you can use the following code
with <a href="/docs/lighteval/pr_1264/en/package_reference/metrics#lighteval.metrics.utils.metric_utils.SampleLevelMetricGrouping">SampleLevelMetricGrouping</a>:`,Le,O,Qe,K,He,ee,$t=`To finish, add the following code so that it adds your metric to our metrics list
when loaded as a module:`,qe,te,Fe,le,Pe,se,De,ne,Tt="You can then give your custom metric to Lighteval by using <code>--custom-tasks path_to_your_file</code> when launching it after adding it to the task config.",Oe,ie,Ke,ae,et,pe,tt,re,lt;return d=new It({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new f({props:{title:"Adding a New Metric",local:"adding-a-new-metric",headingTag:"h1"}}),J=new f({props:{title:"Before You Start",local:"before-you-start",headingTag:"h2"}}),U=new f({props:{title:"Two different types of metrics",local:"two-different-types-of-metrics",headingTag:"h3"}}),C=new f({props:{title:"Sample-Level Metrics",local:"sample-level-metrics",headingTag:"h4"}}),v=new f({props:{title:"Corpus-Level Metrics",local:"corpus-level-metrics",headingTag:"h4"}}),_=new f({props:{title:"Check Existing Metrics",local:"check-existing-metrics",headingTag:"h3"}}),E=new f({props:{title:"Creating a Custom Metric",local:"creating-a-custom-metric",headingTag:"h2"}}),G=new f({props:{title:"Step 1: Create the Metric File",local:"step-1-create-the-metric-file",headingTag:"h3"}}),X=new h({props:{code:"ZnJvbSUyMGFlbnVtJTIwaW1wb3J0JTIwZXh0ZW5kX2VudW0lMEFmcm9tJTIwbGlnaHRldmFsLm1ldHJpY3MlMjBpbXBvcnQlMjBNZXRyaWNz",highlighted:`<span class="hljs-keyword">from</span> aenum <span class="hljs-keyword">import</span> extend_enum
<span class="hljs-keyword">from</span> lighteval.metrics <span class="hljs-keyword">import</span> Metrics`,lang:"python",wrap:!1}}),B=new f({props:{title:"Step 2: Define the Sample-Level Metric",local:"step-2-define-the-sample-level-metric",headingTag:"h3"}}),x=new f({props:{title:"Single Metric Example",local:"single-metric-example",headingTag:"h4"}}),S=new h({props:{code:"ZGVmJTIwY3VzdG9tX21ldHJpYyhkb2MlM0ElMjBEb2MlMkMlMjBtb2RlbF9yZXNwb25zZSUzQSUyME1vZGVsUmVzcG9uc2UpJTIwLSUzRSUyMGJvb2wlM0ElMEElMjAlMjAlMjAlMjByZXNwb25zZSUyMCUzRCUyMG1vZGVsX3Jlc3BvbnNlLmZpbmFsX3RleHQlNUIwJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwcmVzcG9uc2UlMjAlM0QlM0QlMjBkb2MuY2hvaWNlcyU1QmRvYy5nb2xkX2luZGV4JTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">custom_metric</span>(<span class="hljs-params">doc: Doc, model_response: ModelResponse</span>) -&gt; <span class="hljs-built_in">bool</span>:
    response = model_response.final_text[<span class="hljs-number">0</span>]
    <span class="hljs-keyword">return</span> response == doc.choices[doc.gold_index]`,lang:"python",wrap:!1}}),Y=new f({props:{title:"Multiple Metrics Example",local:"multiple-metrics-example",headingTag:"h4"}}),A=new h({props:{code:"ZGVmJTIwY3VzdG9tX21ldHJpYyhkb2MlM0ElMjBEb2MlMkMlMjBtb2RlbF9yZXNwb25zZSUzQSUyME1vZGVsUmVzcG9uc2UpJTIwLSUzRSUyMGRpY3QlM0ElMEElMjAlMjAlMjAlMjByZXNwb25zZSUyMCUzRCUyMG1vZGVsX3Jlc3BvbnNlLmZpbmFsX3RleHQlNUIwJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTdCJTIyYWNjdXJhY3klMjIlM0ElMjByZXNwb25zZSUyMCUzRCUzRCUyMGRvYy5jaG9pY2VzJTVCZG9jLmdvbGRfaW5kZXglNUQlMkMlMjAlMjJvdGhlcl9tZXRyaWMlMjIlM0ElMjAwLjUlN0Q=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">custom_metric</span>(<span class="hljs-params">doc: Doc, model_response: ModelResponse</span>) -&gt; <span class="hljs-built_in">dict</span>:
    response = model_response.final_text[<span class="hljs-number">0</span>]
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;accuracy&quot;</span>: response == doc.choices[doc.gold_index], <span class="hljs-string">&quot;other_metric&quot;</span>: <span class="hljs-number">0.5</span>}`,lang:"python",wrap:!1}}),z=new f({props:{title:"Step 3: Define Aggregation Function (Optional)",local:"step-3-define-aggregation-function-optional",headingTag:"h3"}}),L=new h({props:{code:"ZGVmJTIwYWdnX2Z1bmN0aW9uKGl0ZW1zKSUzQSUwQSUyMCUyMCUyMCUyMGZsYXRfaXRlbXMlMjAlM0QlMjAlNUJpdGVtJTIwZm9yJTIwc3VibGlzdCUyMGluJTIwaXRlbXMlMjBmb3IlMjBpdGVtJTIwaW4lMjBzdWJsaXN0JTVEJTBBJTIwJTIwJTIwJTIwc2NvcmUlMjAlM0QlMjBzdW0oZmxhdF9pdGVtcyklMjAlMkYlMjBsZW4oZmxhdF9pdGVtcyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBzY29yZQ==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">agg_function</span>(<span class="hljs-params">items</span>):
    flat_items = [item <span class="hljs-keyword">for</span> sublist <span class="hljs-keyword">in</span> items <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> sublist]
    score = <span class="hljs-built_in">sum</span>(flat_items) / <span class="hljs-built_in">len</span>(flat_items)
    <span class="hljs-keyword">return</span> score`,lang:"python",wrap:!1}}),Q=new f({props:{title:"Step 4: Create the Metric Object",local:"step-4-create-the-metric-object",headingTag:"h3"}}),H=new f({props:{title:"Single Metric",local:"single-metric",headingTag:"h4"}}),F=new h({props:{code:"bXlfY3VzdG9tX21ldHJpYyUyMCUzRCUyMFNhbXBsZUxldmVsTWV0cmljKCUwQSUyMCUyMCUyMCUyMG1ldHJpY19uYW1lJTNEJTIyY3VzdG9tX2FjY3VyYWN5JTIyJTJDJTBBJTIwJTIwJTIwJTIwaGlnaGVyX2lzX2JldHRlciUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBjYXRlZ29yeSUzRFNhbXBsaW5nTWV0aG9kLkdFTkVSQVRJVkUlMkMlMEElMjAlMjAlMjAlMjBzYW1wbGVfbGV2ZWxfZm4lM0RjdXN0b21fbWV0cmljJTJDJTBBJTIwJTIwJTIwJTIwY29ycHVzX2xldmVsX2ZuJTNEYWdnX2Z1bmN0aW9uJTJDJTBBKQ==",highlighted:`my_custom_metric = SampleLevelMetric(
    metric_name=<span class="hljs-string">&quot;custom_accuracy&quot;</span>,
    higher_is_better=<span class="hljs-literal">True</span>,
    category=SamplingMethod.GENERATIVE,
    sample_level_fn=custom_metric,
    corpus_level_fn=agg_function,
)`,lang:"python",wrap:!1}}),P=new f({props:{title:"Multiple Metrics",local:"multiple-metrics",headingTag:"h4"}}),O=new h({props:{code:"Y3VzdG9tX21ldHJpYyUyMCUzRCUyMFNhbXBsZUxldmVsTWV0cmljR3JvdXBpbmcoJTBBJTIwJTIwJTIwJTIwbWV0cmljX25hbWUlM0QlNUIlMjJhY2N1cmFjeSUyMiUyQyUyMCUyMnJlc3BvbnNlX2xlbmd0aCUyMiUyQyUyMCUyMmNvbmZpZGVuY2UlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBoaWdoZXJfaXNfYmV0dGVyJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWNjdXJhY3klMjIlM0ElMjBUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycmVzcG9uc2VfbGVuZ3RoJTIyJTNBJTIwRmFsc2UlMkMlMjAlMjAlMjMlMjBTaG9ydGVyJTIwcmVzcG9uc2VzJTIwbWlnaHQlMjBiZSUyMGJldHRlciUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbmZpZGVuY2UlMjIlM0ElMjBUcnVlJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwY2F0ZWdvcnklM0RTYW1wbGluZ01ldGhvZC5HRU5FUkFUSVZFJTJDJTBBJTIwJTIwJTIwJTIwc2FtcGxlX2xldmVsX2ZuJTNEY3VzdG9tX21ldHJpYyUyQyUwQSUyMCUyMCUyMCUyMGNvcnB1c19sZXZlbF9mbiUzRCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmFjY3VyYWN5JTIyJTNBJTIwbnAubWVhbiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJlc3BvbnNlX2xlbmd0aCUyMiUzQSUyMG5wLm1lYW4lMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJjb25maWRlbmNlJTIyJTNBJTIwbnAubWVhbiUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSk=",highlighted:`custom_metric = SampleLevelMetricGrouping(
    metric_name=[<span class="hljs-string">&quot;accuracy&quot;</span>, <span class="hljs-string">&quot;response_length&quot;</span>, <span class="hljs-string">&quot;confidence&quot;</span>],
    higher_is_better={
        <span class="hljs-string">&quot;accuracy&quot;</span>: <span class="hljs-literal">True</span>,
        <span class="hljs-string">&quot;response_length&quot;</span>: <span class="hljs-literal">False</span>,  <span class="hljs-comment"># Shorter responses might be better</span>
        <span class="hljs-string">&quot;confidence&quot;</span>: <span class="hljs-literal">True</span>
    },
    category=SamplingMethod.GENERATIVE,
    sample_level_fn=custom_metric,
    corpus_level_fn={
        <span class="hljs-string">&quot;accuracy&quot;</span>: np.mean,
        <span class="hljs-string">&quot;response_length&quot;</span>: np.mean,
        <span class="hljs-string">&quot;confidence&quot;</span>: np.mean,
    },
)`,lang:"python",wrap:!1}}),K=new f({props:{title:"Step 5: Register the Metric",local:"step-5-register-the-metric",headingTag:"h3"}}),te=new h({props:{code:"JTIzJTIwQWRkcyUyMHRoZSUyMG1ldHJpYyUyMHRvJTIwdGhlJTIwbWV0cmljJTIwbGlzdCElMEFleHRlbmRfZW51bShNZXRyaWNzJTJDJTIwJTIyQ1VTVE9NX0FDQ1VSQUNZJTIyJTJDJTIwbXlfY3VzdG9tX21ldHJpYyklMEElMEFpZiUyMF9fbmFtZV9fJTIwJTNEJTNEJTIwJTIyX19tYWluX18lMjIlM0ElMEElMjAlMjAlMjAlMjBwcmludCglMjJJbXBvcnRlZCUyMG1ldHJpYyUyMik=",highlighted:`<span class="hljs-comment"># Adds the metric to the metric list!</span>
extend_enum(Metrics, <span class="hljs-string">&quot;CUSTOM_ACCURACY&quot;</span>, my_custom_metric)

<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Imported metric&quot;</span>)`,lang:"python",wrap:!1}}),le=new f({props:{title:"Using Your Custom Metric",local:"using-your-custom-metric",headingTag:"h2"}}),se=new f({props:{title:"With Custom Tasks",local:"with-custom-tasks",headingTag:"h3"}}),ie=new h({props:{code:"bGlnaHRldmFsJTIwYWNjZWxlcmF0ZSUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMm1vZGVsX25hbWUlM0RvcGVuYWktY29tbXVuaXR5JTJGZ3B0MiUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMnRydXRoZnVscWElM0FtYyUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tY3VzdG9tLXRhc2tzJTIwcGF0aF90b195b3VyX21ldHJpY19maWxlLnB5",highlighted:`lighteval accelerate \\
    <span class="hljs-string">&quot;model_name=openai-community/gpt2&quot;</span> \\
    <span class="hljs-string">&quot;truthfulqa:mc&quot;</span> \\
    --custom-tasks path_to_your_metric_file.py`,lang:"bash",wrap:!1}}),ae=new h({props:{code:"ZnJvbSUyMGxpZ2h0ZXZhbC50YXNrcy5saWdodGV2YWxfdGFzayUyMGltcG9ydCUyMExpZ2h0ZXZhbFRhc2tDb25maWclMEElMEF0YXNrJTIwJTNEJTIwTGlnaHRldmFsVGFza0NvbmZpZyglMEElMjAlMjAlMjAlMjBuYW1lJTNEJTIybXlfY3VzdG9tX3Rhc2slMjIlMkMlMEElMjAlMjAlMjAlMjBtZXRyaWMlM0QlNUJteV9jdXN0b21fbWV0cmljJTVEJTJDJTIwJTIwJTIzJTIwVXNlJTIweW91ciUyMGN1c3RvbSUyMG1ldHJpYyUyMGhlcmUlMEElMjAlMjAlMjAlMjBwcm9tcHRfZnVuY3Rpb24lM0RteV9wcm9tcHRfZnVuY3Rpb24lMkMlMEElMjAlMjAlMjAlMjBoZl9yZXBvJTNEJTIybXlfZGF0YXNldCUyMiUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3BsaXRzJTNEJTVCJTIydGVzdCUyMiU1RCUwQSk=",highlighted:`<span class="hljs-keyword">from</span> lighteval.tasks.lighteval_task <span class="hljs-keyword">import</span> LightevalTaskConfig

task = LightevalTaskConfig(
    name=<span class="hljs-string">&quot;my_custom_task&quot;</span>,
    metric=[my_custom_metric],  <span class="hljs-comment"># Use your custom metric here</span>
    prompt_function=my_prompt_function,
    hf_repo=<span class="hljs-string">&quot;my_dataset&quot;</span>,
    evaluation_splits=[<span class="hljs-string">&quot;test&quot;</span>]
)`,lang:"python",wrap:!1}}),pe=new _t({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/adding-a-new-metric.mdx"}}),{c(){g=M("meta"),ce=n(),me=M("p"),oe=n(),a(d.$$.fragment),Me=n(),a(w.$$.fragment),ue=n(),a(J.$$.fragment),fe=n(),a(U.$$.fragment),ye=n(),j=M("p"),j.textContent=it,ge=n(),a(C.$$.fragment),he=n(),b=M("ul"),b.innerHTML=at,$e=n(),a(v.$$.fragment),Te=n(),I=M("ul"),I.innerHTML=pt,de=n(),a(_.$$.fragment),we=n(),k=M("p"),k.innerHTML=mt,Je=n(),V=M("p"),V.innerHTML=rt,Ue=n(),$=M("blockquote"),$.innerHTML=ct,je=n(),T=M("blockquote"),T.innerHTML=ot,Ce=n(),a(E.$$.fragment),be=n(),a(G.$$.fragment),ve=n(),Z=M("p"),Z.textContent=Mt,Ie=n(),a(X.$$.fragment),_e=n(),a(B.$$.fragment),ke=n(),R=M("p"),R.innerHTML=ut,Ve=n(),a(x.$$.fragment),Ee=n(),a(S.$$.fragment),Ge=n(),a(Y.$$.fragment),Ze=n(),N=M("p"),N.textContent=ft,Xe=n(),a(A.$$.fragment),Be=n(),a(z.$$.fragment),Re=n(),W=M("p"),W.innerHTML=yt,xe=n(),a(L.$$.fragment),Se=n(),a(Q.$$.fragment),Ye=n(),a(H.$$.fragment),Ne=n(),q=M("p"),q.innerHTML=gt,Ae=n(),a(F.$$.fragment),ze=n(),a(P.$$.fragment),We=n(),D=M("p"),D.innerHTML=ht,Le=n(),a(O.$$.fragment),Qe=n(),a(K.$$.fragment),He=n(),ee=M("p"),ee.textContent=$t,qe=n(),a(te.$$.fragment),Fe=n(),a(le.$$.fragment),Pe=n(),a(se.$$.fragment),De=n(),ne=M("p"),ne.innerHTML=Tt,Oe=n(),a(ie.$$.fragment),Ke=n(),a(ae.$$.fragment),et=n(),a(pe.$$.fragment),tt=n(),re=M("p"),this.h()},l(e){const t=bt("svelte-u9bgzb",document.head);g=u(t,"META",{name:!0,content:!0}),t.forEach(l),ce=i(e),me=u(e,"P",{}),dt(me).forEach(l),oe=i(e),p(d.$$.fragment,e),Me=i(e),p(w.$$.fragment,e),ue=i(e),p(J.$$.fragment,e),fe=i(e),p(U.$$.fragment,e),ye=i(e),j=u(e,"P",{"data-svelte-h":!0}),y(j)!=="svelte-1jnixsg"&&(j.textContent=it),ge=i(e),p(C.$$.fragment,e),he=i(e),b=u(e,"UL",{"data-svelte-h":!0}),y(b)!=="svelte-1nwago0"&&(b.innerHTML=at),$e=i(e),p(v.$$.fragment,e),Te=i(e),I=u(e,"UL",{"data-svelte-h":!0}),y(I)!=="svelte-15e9ylm"&&(I.innerHTML=pt),de=i(e),p(_.$$.fragment,e),we=i(e),k=u(e,"P",{"data-svelte-h":!0}),y(k)!=="svelte-vpq2qz"&&(k.innerHTML=mt),Je=i(e),V=u(e,"P",{"data-svelte-h":!0}),y(V)!=="svelte-xdscpl"&&(V.innerHTML=rt),Ue=i(e),$=u(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),y($)!=="svelte-1nxhxit"&&($.innerHTML=ct),je=i(e),T=u(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),y(T)!=="svelte-3wrpqo"&&(T.innerHTML=ot),Ce=i(e),p(E.$$.fragment,e),be=i(e),p(G.$$.fragment,e),ve=i(e),Z=u(e,"P",{"data-svelte-h":!0}),y(Z)!=="svelte-rk8zi4"&&(Z.textContent=Mt),Ie=i(e),p(X.$$.fragment,e),_e=i(e),p(B.$$.fragment,e),ke=i(e),R=u(e,"P",{"data-svelte-h":!0}),y(R)!=="svelte-uw89ed"&&(R.innerHTML=ut),Ve=i(e),p(x.$$.fragment,e),Ee=i(e),p(S.$$.fragment,e),Ge=i(e),p(Y.$$.fragment,e),Ze=i(e),N=u(e,"P",{"data-svelte-h":!0}),y(N)!=="svelte-1vuyefu"&&(N.textContent=ft),Xe=i(e),p(A.$$.fragment,e),Be=i(e),p(z.$$.fragment,e),Re=i(e),W=u(e,"P",{"data-svelte-h":!0}),y(W)!=="svelte-n3kro3"&&(W.innerHTML=yt),xe=i(e),p(L.$$.fragment,e),Se=i(e),p(Q.$$.fragment,e),Ye=i(e),p(H.$$.fragment,e),Ne=i(e),q=u(e,"P",{"data-svelte-h":!0}),y(q)!=="svelte-13wakgs"&&(q.innerHTML=gt),Ae=i(e),p(F.$$.fragment,e),ze=i(e),p(P.$$.fragment,e),We=i(e),D=u(e,"P",{"data-svelte-h":!0}),y(D)!=="svelte-1ouo1u2"&&(D.innerHTML=ht),Le=i(e),p(O.$$.fragment,e),Qe=i(e),p(K.$$.fragment,e),He=i(e),ee=u(e,"P",{"data-svelte-h":!0}),y(ee)!=="svelte-1bbv5uf"&&(ee.textContent=$t),qe=i(e),p(te.$$.fragment,e),Fe=i(e),p(le.$$.fragment,e),Pe=i(e),p(se.$$.fragment,e),De=i(e),ne=u(e,"P",{"data-svelte-h":!0}),y(ne)!=="svelte-o8irch"&&(ne.innerHTML=Tt),Oe=i(e),p(ie.$$.fragment,e),Ke=i(e),p(ae.$$.fragment,e),et=i(e),p(pe.$$.fragment,e),tt=i(e),re=u(e,"P",{}),dt(re).forEach(l),this.h()},h(){st(g,"name","hf:doc:metadata"),st(g,"content",Vt),st($,"class","tip"),st(T,"class","warning")},m(e,t){vt(document.head,g),s(e,ce,t),s(e,me,t),s(e,oe,t),m(d,e,t),s(e,Me,t),m(w,e,t),s(e,ue,t),m(J,e,t),s(e,fe,t),m(U,e,t),s(e,ye,t),s(e,j,t),s(e,ge,t),m(C,e,t),s(e,he,t),s(e,b,t),s(e,$e,t),m(v,e,t),s(e,Te,t),s(e,I,t),s(e,de,t),m(_,e,t),s(e,we,t),s(e,k,t),s(e,Je,t),s(e,V,t),s(e,Ue,t),s(e,$,t),s(e,je,t),s(e,T,t),s(e,Ce,t),m(E,e,t),s(e,be,t),m(G,e,t),s(e,ve,t),s(e,Z,t),s(e,Ie,t),m(X,e,t),s(e,_e,t),m(B,e,t),s(e,ke,t),s(e,R,t),s(e,Ve,t),m(x,e,t),s(e,Ee,t),m(S,e,t),s(e,Ge,t),m(Y,e,t),s(e,Ze,t),s(e,N,t),s(e,Xe,t),m(A,e,t),s(e,Be,t),m(z,e,t),s(e,Re,t),s(e,W,t),s(e,xe,t),m(L,e,t),s(e,Se,t),m(Q,e,t),s(e,Ye,t),m(H,e,t),s(e,Ne,t),s(e,q,t),s(e,Ae,t),m(F,e,t),s(e,ze,t),m(P,e,t),s(e,We,t),s(e,D,t),s(e,Le,t),m(O,e,t),s(e,Qe,t),m(K,e,t),s(e,He,t),s(e,ee,t),s(e,qe,t),m(te,e,t),s(e,Fe,t),m(le,e,t),s(e,Pe,t),m(se,e,t),s(e,De,t),s(e,ne,t),s(e,Oe,t),m(ie,e,t),s(e,Ke,t),m(ae,e,t),s(e,et,t),m(pe,e,t),s(e,tt,t),s(e,re,t),lt=!0},p:Jt,i(e){lt||(r(d.$$.fragment,e),r(w.$$.fragment,e),r(J.$$.fragment,e),r(U.$$.fragment,e),r(C.$$.fragment,e),r(v.$$.fragment,e),r(_.$$.fragment,e),r(E.$$.fragment,e),r(G.$$.fragment,e),r(X.$$.fragment,e),r(B.$$.fragment,e),r(x.$$.fragment,e),r(S.$$.fragment,e),r(Y.$$.fragment,e),r(A.$$.fragment,e),r(z.$$.fragment,e),r(L.$$.fragment,e),r(Q.$$.fragment,e),r(H.$$.fragment,e),r(F.$$.fragment,e),r(P.$$.fragment,e),r(O.$$.fragment,e),r(K.$$.fragment,e),r(te.$$.fragment,e),r(le.$$.fragment,e),r(se.$$.fragment,e),r(ie.$$.fragment,e),r(ae.$$.fragment,e),r(pe.$$.fragment,e),lt=!0)},o(e){c(d.$$.fragment,e),c(w.$$.fragment,e),c(J.$$.fragment,e),c(U.$$.fragment,e),c(C.$$.fragment,e),c(v.$$.fragment,e),c(_.$$.fragment,e),c(E.$$.fragment,e),c(G.$$.fragment,e),c(X.$$.fragment,e),c(B.$$.fragment,e),c(x.$$.fragment,e),c(S.$$.fragment,e),c(Y.$$.fragment,e),c(A.$$.fragment,e),c(z.$$.fragment,e),c(L.$$.fragment,e),c(Q.$$.fragment,e),c(H.$$.fragment,e),c(F.$$.fragment,e),c(P.$$.fragment,e),c(O.$$.fragment,e),c(K.$$.fragment,e),c(te.$$.fragment,e),c(le.$$.fragment,e),c(se.$$.fragment,e),c(ie.$$.fragment,e),c(ae.$$.fragment,e),c(pe.$$.fragment,e),lt=!1},d(e){e&&(l(ce),l(me),l(oe),l(Me),l(ue),l(fe),l(ye),l(j),l(ge),l(he),l(b),l($e),l(Te),l(I),l(de),l(we),l(k),l(Je),l(V),l(Ue),l($),l(je),l(T),l(Ce),l(be),l(ve),l(Z),l(Ie),l(_e),l(ke),l(R),l(Ve),l(Ee),l(Ge),l(Ze),l(N),l(Xe),l(Be),l(Re),l(W),l(xe),l(Se),l(Ye),l(Ne),l(q),l(Ae),l(ze),l(We),l(D),l(Le),l(Qe),l(He),l(ee),l(qe),l(Fe),l(Pe),l(De),l(ne),l(Oe),l(Ke),l(et),l(tt),l(re)),l(g),o(d,e),o(w,e),o(J,e),o(U,e),o(C,e),o(v,e),o(_,e),o(E,e),o(G,e),o(X,e),o(B,e),o(x,e),o(S,e),o(Y,e),o(A,e),o(z,e),o(L,e),o(Q,e),o(H,e),o(F,e),o(P,e),o(O,e),o(K,e),o(te,e),o(le,e),o(se,e),o(ie,e),o(ae,e),o(pe,e)}}}const Vt='{"title":"Adding a New Metric","local":"adding-a-new-metric","sections":[{"title":"Before You Start","local":"before-you-start","sections":[{"title":"Two different types of metrics","local":"two-different-types-of-metrics","sections":[{"title":"Sample-Level Metrics","local":"sample-level-metrics","sections":[],"depth":4},{"title":"Corpus-Level Metrics","local":"corpus-level-metrics","sections":[],"depth":4}],"depth":3},{"title":"Check Existing Metrics","local":"check-existing-metrics","sections":[],"depth":3}],"depth":2},{"title":"Creating a Custom Metric","local":"creating-a-custom-metric","sections":[{"title":"Step 1: Create the Metric File","local":"step-1-create-the-metric-file","sections":[],"depth":3},{"title":"Step 2: Define the Sample-Level Metric","local":"step-2-define-the-sample-level-metric","sections":[{"title":"Single Metric Example","local":"single-metric-example","sections":[],"depth":4},{"title":"Multiple Metrics Example","local":"multiple-metrics-example","sections":[],"depth":4}],"depth":3},{"title":"Step 3: Define Aggregation Function (Optional)","local":"step-3-define-aggregation-function-optional","sections":[],"depth":3},{"title":"Step 4: Create the Metric Object","local":"step-4-create-the-metric-object","sections":[{"title":"Single Metric","local":"single-metric","sections":[],"depth":4},{"title":"Multiple Metrics","local":"multiple-metrics","sections":[],"depth":4}],"depth":3},{"title":"Step 5: Register the Metric","local":"step-5-register-the-metric","sections":[],"depth":3}],"depth":2},{"title":"Using Your Custom Metric","local":"using-your-custom-metric","sections":[{"title":"With Custom Tasks","local":"with-custom-tasks","sections":[],"depth":3}],"depth":2}],"depth":1}';function Et(nt){return Ut(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rt extends jt{constructor(g){super(),Ct(this,g,Et,kt,wt,{})}}export{Rt as component};