lighteval/pr_706/en/_app/immutable/nodes/5.2f87bf9a.js

import{s as dl,o as Al,n as bl}from"../chunks/scheduler.7da89386.js";import{S as gl,i as fl,g as y,s as a,r as B,A as Gl,h as U,f as M,c as s,j as hl,u as W,x as i,k as nl,y as Zl,a as e,v as k,d as Q,t as S,w as F}from"../chunks/index.20910acc.js";import{T as vl}from"../chunks/Tip.53e22153.js";import{C as Il}from"../chunks/CodeBlock.143bd81e.js";import{H as yl,E as Bl}from"../chunks/index.3af8f81c.js";function Wl(z){let n,T='All <a href="/docs/lighteval/pr_706/en/package_reference/tasks#lighteval.tasks.lighteval_task.LightevalTaskConfig">LightevalTaskConfig</a> parameters are strongly typed, including the inputs to the template function. Make sure to take advantage of your IDE’s functionality to make it easier to correctly fill these parameters.';return{c(){n=y("p"),n.innerHTML=T},l(o){n=U(o,"P",{"data-svelte-h":!0}),i(n)!=="svelte-1jban9e"&&(n.innerHTML=T)},m(o,q){e(o,n,q)},p:bl,d(o){o&&M(n)}}}function kl(z){let n,T,o,q,C,R,r,H,w,Ul="We define 19 <code>literals</code>, basic keywords or punctuation signs used when creating evaluation prompts in an automatic manner, such as <code>yes</code>, <code>no</code>, <code>because</code>, etc.",X,m,il="We welcome translations in your language!",Y,p,ol="To contribute, you’ll need to",E,c,jl='<li>Open the <a href="https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/utils/translation_literals.py" rel="nofollow">translation_literals</a> file</li> <li>Edit the file to add or expand the literal for your language of interest.</li>',_,h,V,j,Jl="<li>Open a PR with your modifications! And voilà!</li>",$,I,x,d,ul='You should first read our guide on <a href="adding-a-custom-task">adding a custom task</a>, to better understand the different parameters we use.',L,A,Tl='Then, you should take a look at the current <a href="https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/multilingual/tasks.py" rel="nofollow">multilingual tasks</a> file, to understand how they are defined. For multilingual evaluations the <code>prompt_function</code> should be implemented by language-adapted template. The template will take care of correct formatting, correct and consistent usage of language adjusted prompt anchors (e.g Question/Answer) and punctuation.',P,b,Cl='Browse the list of all templates <a href="https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/templates" rel="nofollow">here</a> to see which are the most adapted to your own task.',D,g,rl="Then, when ready, to define your own task, you should:",O,f,wl='<li>create a Python file as indicated in the above guide</li> <li>import the relevant templates for your task type (XNLI, Copa, Multiple choice, Question Answering, etc)</li> <li>define one or a list of tasks for each relevant language and evaluation formulation (for multichoice) using our parametrizable <a href="/docs/lighteval/pr_706/en/package_reference/tasks#lighteval.tasks.lighteval_task.LightevalTaskConfig">LightevalTaskConfig</a> class</li>',K,G,ll,J,ml="<li>then, you can go back to the guide to test if your task is correctly implemented!</li>",tl,u,Ml,Z,pl="Once everything is good, open a PR, and we’ll be happy to review it!",el,v,al,N,sl;return C=new yl({props:{title:"Contributing to multilingual evaluations",local:"contributing-to-multilingual-evaluations",headingTag:"h1"}}),r=new yl({props:{title:"Contributing a small translation",local:"contributing-a-small-translation",headingTag:"h2"}}),h=new Il({props:{code:"JTIwJTIwJTIwJTIwTGFuZ3VhZ2UuRU5HTElTSCUzQSUyMFRyYW5zbGF0aW9uTGl0ZXJhbHMoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFuZ3VhZ2UlM0RMYW5ndWFnZS5FTkdMSVNIJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcXVlc3Rpb25fd29yZCUzRCUyMnF1ZXN0aW9uJTIyJTJDJTIwJTIzJTIwVXNhZ2UlM0ElMjAlMjJRdWVzdGlvbiUzQSUyMEhvdyUyMGFyZSUyMHlvdSUzRiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFuc3dlciUzRCUyMmFuc3dlciUyMiUyQyUyMCUyMyUyMFVzYWdlJTNBJTIwJTIyQW5zd2VyJTNBJTIwSSUyMGFtJTIwZmluZSUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbmZpcm1hdGlvbl93b3JkJTNEJTIycmlnaHQlMjIlMkMlMjAlMjMlMjBVc2FnZSUzQSUyMCUyMkhlJTIwaXMlMjBzbWFydCUyQyUyMHJpZ2h0JTNGJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIweWVzJTNEJTIyeWVzJTIyJTJDJTIwJTIzJTIwVXNhZ2UlM0ElMjAlMjJZZXMlMkMlMjBoZSUyMGlzJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbm8lM0QlMjJubyUyMiUyQyUyMCUyMyUyMFVzYWdlJTNBJTIwJTIyTm8lMkMlMjBoZSUyMGlzJTIwbm90JTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWxzbyUzRCUyMmFsc28lMjIlMkMlMjAlMjMlMjBVc2FnZSUzQSUyMCUyMkFsc28lMkMlMjBzaGUlMjBpcyUyMHNtYXJ0LiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNhdXNlX3dvcmQlM0QlMjJiZWNhdXNlJTIyJTJDJTIwJTIzJTIwVXNhZ2UlM0ElMjAlMjJTaGUlMjBpcyUyMHNtYXJ0JTJDJTIwYmVjYXVzZSUyMHNoZSUyMGlzJTIwdGFsbCUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVmZmVjdF93b3JkJTNEJTIydGhlcmVmb3JlJTIyJTJDJTIwJTIzJTIwVXNhZ2UlM0ElMjAlMjJIZSUyMGlzJTIwdGFsbCUyMHRoZXJlZm9yZSUyMGhlJTIwaXMlMjBzbWFydCUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9yX3dvcmQlM0QlMjJvciUyMiUyQyUyMCUyMyUyMFVzYWdlJTNBJTIwJTIySGUlMjBpcyUyMHRhbGwlMjBvciUyMHNtYWxsJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJ1ZSUzRCUyMnRydWUlMjIlMkMlMjAlMjMlMjBVc2FnZSUzQSUyMCUyMkhlJTIwaXMlMjBzbWFydCUyQyUyMHRydWUlMkMlMjBmYWxzZSUyMG9yJTIwbmVpdGhlciUzRiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZhbHNlJTNEJTIyZmFsc2UlMjIlMkMlMjAlMjMlMjBVc2FnZSUzQSUyMCUyMkhlJTIwaXMlMjBzbWFydCUyQyUyMHRydWUlMkMlMjBmYWxzZSUyMG9yJTIwbmVpdGhlciUzRiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG5laXRoZXIlM0QlMjJuZWl0aGVyJTIyJTJDJTIwJTIzJTIwVXNhZ2UlM0ElMjAlMjJIZSUyMGlzJTIwc21hcnQlMkMlMjB0cnVlJTJDJTIwZmFsc2UlMjBvciUyMG5laXRoZXIlM0YlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBQdW5jdHVhdGlvbiUyMGFuZCUyMHNwYWNpbmclM0ElMjBvbmx5JTIwYWRqdXN0JTIwaWYlMjB5b3VyJTIwbGFuZ3VhZ2UlMjB1c2VzJTIwc29tZXRoaW5nJTIwZGlmZmVyZW50JTIwdGhhbiUyMGluJTIwRW5nbGlzaCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZ1bGxfc3RvcCUzRCUyMi4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb21tYSUzRCUyMiUyQyUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXN0aW9uX21hcmslM0QlMjIlM0YlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBleGNsYW1hdGlvbl9tYXJrJTNEJTIyISUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHdvcmRfc3BhY2UlM0QlMjIlMjAlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZW50ZW5jZV9zcGFjZSUzRCUyMiUyMCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbG9uJTNEJTIyJTNBJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwVGhlJTIwZmlyc3QlMjBjaGFyYWN0ZXJzJTIwb2YlMjB5b3VyJTIwYWxwaGFiZXQlMjB1c2VkJTIwaW4lMjBlbnVtZXJhdGlvbnMlMkMlMjBpZiUyMGRpZmZlcmVudCUyMGZyb20lMjBFbmdsaXNoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5kaWNlcyUzRCU1QiUyMkElMjIlMkMlMjAlMjJCJTIyJTJDJTIwJTIyQyUyMiUyQyUyMC4uLiU1RCUwQSUyMCUyMCUyMCUyMCk=",highlighted:`    Language.ENGLISH: TranslationLiterals(
        language=Language.ENGLISH,
        question_word=<span class="hljs-string">&quot;question&quot;</span>, <span class="hljs-comment"># Usage: &quot;Question: How are you?&quot;</span>
        answer=<span class="hljs-string">&quot;answer&quot;</span>, <span class="hljs-comment"># Usage: &quot;Answer: I am fine&quot;</span>
        confirmation_word=<span class="hljs-string">&quot;right&quot;</span>, <span class="hljs-comment"># Usage: &quot;He is smart, right?&quot;</span>
        yes=<span class="hljs-string">&quot;yes&quot;</span>, <span class="hljs-comment"># Usage: &quot;Yes, he is&quot;</span>
        no=<span class="hljs-string">&quot;no&quot;</span>, <span class="hljs-comment"># Usage: &quot;No, he is not&quot;</span>
        also=<span class="hljs-string">&quot;also&quot;</span>, <span class="hljs-comment"># Usage: &quot;Also, she is smart.&quot;</span>
        cause_word=<span class="hljs-string">&quot;because&quot;</span>, <span class="hljs-comment"># Usage: &quot;She is smart, because she is tall&quot;</span>
        effect_word=<span class="hljs-string">&quot;therefore&quot;</span>, <span class="hljs-comment"># Usage: &quot;He is tall therefore he is smart&quot;</span>
        or_word=<span class="hljs-string">&quot;or&quot;</span>, <span class="hljs-comment"># Usage: &quot;He is tall or small&quot;</span>
        true=<span class="hljs-string">&quot;true&quot;</span>, <span class="hljs-comment"># Usage: &quot;He is smart, true, false or neither?&quot;</span>
        false=<span class="hljs-string">&quot;false&quot;</span>, <span class="hljs-comment"># Usage: &quot;He is smart, true, false or neither?&quot;</span>
        neither=<span class="hljs-string">&quot;neither&quot;</span>, <span class="hljs-comment"># Usage: &quot;He is smart, true, false or neither?&quot;</span>
        <span class="hljs-comment"># Punctuation and spacing: only adjust if your language uses something different than in English</span>
        full_stop=<span class="hljs-string">&quot;.&quot;</span>,
        comma=<span class="hljs-string">&quot;,&quot;</span>,
        question_mark=<span class="hljs-string">&quot;?&quot;</span>,
        exclamation_mark=<span class="hljs-string">&quot;!&quot;</span>,
        word_space=<span class="hljs-string">&quot; &quot;</span>,
        sentence_space=<span class="hljs-string">&quot; &quot;</span>,
        colon=<span class="hljs-string">&quot;:&quot;</span>,
        <span class="hljs-comment"># The first characters of your alphabet used in enumerations, if different from English</span>
        indices=[<span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;B&quot;</span>, <span class="hljs-string">&quot;C&quot;</span>, ...]
    )`,wrap:!1}}),I=new yl({props:{title:"Contributing a new multilingual task",local:"contributing-a-new-multilingual-task",headingTag:"h2"}}),G=new Il({props:{code:"eW91cl90YXNrcyUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMExpZ2h0ZXZhbFRhc2tDb25maWcoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwTmFtZSUyMG9mJTIweW91ciUyMGV2YWx1YXRpb24lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBuYW1lJTNEZiUyMmV2YWxuYW1lXyU3Qmxhbmd1YWdlLnZhbHVlJTdEXyU3QmZvcm11bGF0aW9uLm5hbWUubG93ZXIoKSU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFRoZSUyMGV2YWx1YXRpb24lMjBpcyUyMGNvbW11bml0eSUyMGNvbnRyaWJ1dGVkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VpdGUlM0QlNUIlMjJjb21tdW5pdHklMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBUaGlzJTIwd2lsbCUyMGF1dG9tYXRpY2FsbHklMjBnZXQlMjB0aGUlMjBjb3JyZWN0JTIwbWV0cmljcyUyMGZvciUyMHlvdXIlMjBjaG9zZW4lMjBmb3JtdWxhdGlvbiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1ldHJpYyUzRGdldF9tZXRyaWNzX2Zvcl9mb3JtdWxhdGlvbiglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3JtdWxhdGlvbiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvZ2xpa2VsaWhvb2RfYWNjX21ldHJpYyhub3JtYWxpemF0aW9uJTNETm9uZSklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb2dsaWtlbGlob29kX2FjY19tZXRyaWMobm9ybWFsaXphdGlvbiUzRExvZ1Byb2JUb2tlbk5vcm0oKSklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb2dsaWtlbGlob29kX2FjY19tZXRyaWMobm9ybWFsaXphdGlvbiUzRExvZ1Byb2JDaGFyTm9ybSgpKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBJbiUyMHRoaXMlMjBmdW5jdGlvbiUyQyUyMHlvdSUyMGNob29zZSUyMHdoaWNoJTIwdGVtcGxhdGUlMjB0byUyMGZvbGxvdyUyMGFuZCUyMGZvciUyMHdoaWNoJTIwbGFuZ3VhZ2UlMjBhbmQlMjBmb3JtdWxhdGlvbiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByb21wdF9mdW5jdGlvbiUzRGdldF90ZW1wbGF0ZV9wcm9tcHRfZnVuY3Rpb24oJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFuZ3VhZ2UlM0RsYW5ndWFnZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMHRoZW4lMjB1c2UlMjB0aGUlMjBhZGFwdGVyJTIwdG8lMjBkZWZpbmUlMjB0aGUlMjBtYXBwaW5nJTIwYmV0d2VlbiUyMHRoZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGtleXMlMjBvZiUyMHRoZSUyMHRlbXBsYXRlJTIwKGxlZnQpJTJDJTIwYW5kJTIwdGhlJTIwa2V5cyUyMG9mJTIweW91ciUyMGRhdGFzZXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjAocmlnaHQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwVG8lMjBrbm93JTIwd2hpY2glMjB0ZW1wbGF0ZSUyMGtleXMlMjBhcmUlMjByZXF1aXJlZCUyMGFuZCUyMGF2YWlsYWJsZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGNvbnN1bHQlMjB0aGUlMjBhcHByb3ByaWF0ZSUyMGFkYXB0ZXIlMjB0eXBlJTIwYW5kJTIwZG9jLXN0cmluZy4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhZGFwdGVyJTNEbGFtYmRhJTIwbGluZSUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmtleSUyMiUzQSUyMGxpbmUlNUIlMjJyZWxldmFudF9rZXklMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAuLi4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3JtdWxhdGlvbiUzRGZvcm11bGF0aW9uJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFlvdSUyMGNhbiUyMGFsc28lMjBhZGQlMjBzcGVjaWZpYyUyMGZpbHRlcnMlMjB0byUyMHJlbW92ZSUyMGlycmVsZXZhbnQlMjBzYW1wbGVzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaGZfZmlsdGVyJTNEbGFtYmRhJTIwbGluZSUzQSUyMGxpbmUlNUIlMjJsYWJlbCUyMiU1RCUyMGluJTIwJTNDY29uZGl0aW9uJTNFJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwWW91JTIwdGhlbiUyMHNlbGVjdCUyMHlvdXIlMjBodWdnaW5nZmFjZSUyMGRhdGFzZXQlMjBhcyUyMHdlbGwlMjBhcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMHRoZSUyMHNwbGl0cyUyMGF2YWlsYWJsZSUyMGZvciUyMGV2YWx1YXRpb24lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBoZl9yZXBvJTNEJTNDZGF0YXNldCUzRSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGhmX3N1YnNldCUzRCUzQ3N1YnNldCUzRSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3BsaXRzJTNEJTVCJTIydHJhaW4lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBoZl9hdmFpbF9zcGxpdHMlM0QlNUIlMjJ0cmFpbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjBmb3IlMjBsYW5ndWFnZSUyMGluJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTGFuZ3VhZ2UuWU9VUl9MQU5HVUFHRSUyQyUyMC4uLiUwQSUyMCUyMCUyMCUyMCU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGZvcm11bGF0aW9uJTIwaW4lMjAlNUJNQ0ZGb3JtdWxhdGlvbigpJTJDJTIwQ0ZGb3JtdWxhdGlvbigpJTJDJTIwSHlicmlkRm9ybXVsYXRpb24oKSU1RCUwQSU1RA==",highlighted:`your_tasks = [
    LightevalTaskConfig(
        <span class="hljs-comment"># Name of your evaluation</span>
        name=<span class="hljs-string">f&quot;evalname_<span class="hljs-subst">{language.value}</span>_<span class="hljs-subst">{formulation.name.lower()}</span>&quot;</span>,
        <span class="hljs-comment"># The evaluation is community contributed</span>
        suite=[<span class="hljs-string">&quot;community&quot;</span>],
        <span class="hljs-comment"># This will automatically get the correct metrics for your chosen formulation</span>
        metric=get_metrics_for_formulation(
            formulation,
            [
                loglikelihood_acc_metric(normalization=<span class="hljs-literal">None</span>),
                loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
                loglikelihood_acc_metric(normalization=LogProbCharNorm()),
            ],
        ),
        <span class="hljs-comment"># In this function, you choose which template to follow and for which language and formulation</span>
        prompt_function=get_template_prompt_function(
            language=language,
            <span class="hljs-comment"># then use the adapter to define the mapping between the</span>
            <span class="hljs-comment"># keys of the template (left), and the keys of your dataset</span>
            <span class="hljs-comment"># (right)</span>
            <span class="hljs-comment"># To know which template keys are required and available,</span>
            <span class="hljs-comment"># consult the appropriate adapter type and doc-string.</span>
            adapter=<span class="hljs-keyword">lambda</span> line: {
                <span class="hljs-string">&quot;key&quot;</span>: line[<span class="hljs-string">&quot;relevant_key&quot;</span>],
                ...
            },
            formulation=formulation,
        ),
        <span class="hljs-comment"># You can also add specific filters to remove irrelevant samples</span>
        hf_filter=<span class="hljs-keyword">lambda</span> line: line[<span class="hljs-string">&quot;label&quot;</span>] <span class="hljs-keyword">in</span> &lt;condition&gt;,
        <span class="hljs-comment"># You then select your huggingface dataset as well as</span>
        <span class="hljs-comment"># the splits available for evaluation</span>
        hf_repo=&lt;dataset&gt;,
        hf_subset=&lt;subset&gt;,
        evaluation_splits=[<span class="hljs-string">&quot;train&quot;</span>],
        hf_avail_splits=[<span class="hljs-string">&quot;train&quot;</span>],
    )
    <span class="hljs-keyword">for</span> language <span class="hljs-keyword">in</span> [
        Language.YOUR_LANGUAGE, ...
    ]
    <span class="hljs-keyword">for</span> formulation <span class="hljs-keyword">in</span> [MCFFormulation(), CFFormulation(), HybridFormulation()]
]`,wrap:!1}}),u=new vl({props:{warning:!1,$$slots:{default:[Wl]},$$scope:{ctx:z}}}),v=new Bl({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/contributing-to-multilingual-evaluations.mdx"}}),{c(){n=y("meta"),T=a(),o=y("p"),q=a(),B(C.$$.fragment),R=a(),B(r.$$.fragment),H=a(),w=y("p"),w.innerHTML=Ul,X=a(),m=y("p"),m.textContent=il,Y=a(),p=y("p"),p.textContent=ol,E=a(),c=y("ol"),c.innerHTML=jl,_=a(),B(h.$$.fragment),V=a(),j=y("ol"),j.innerHTML=Jl,$=a(),B(I.$$.fragment),x=a(),d=y("p"),d.innerHTML=ul,L=a(),A=y("p"),A.innerHTML=Tl,P=a(),b=y("p"),b.innerHTML=Cl,D=a(),g=y("p"),g.textContent=rl,O=a(),f=y("ol"),f.innerHTML=wl,K=a(),B(G.$$.fragment),ll=a(),J=y("ol"),J.innerHTML=ml,tl=a(),B(u.$$.fragment),Ml=a(),Z=y("p"),Z.textContent=pl,el=a(),B(v.$$.fragment),al=a(),N=y("p"),this.h()},l(l){const t=Gl("svelte-u9bgzb",document.head);n=U(t,"META",{name:!0,content:!0}),t.forEach(M),T=s(l),o=U(l,"P",{}),hl(o).forEach(M),q=s(l),W(C.$$.fragment,l),R=s(l),W(r.$$.fragment,l),H=s(l),w=U(l,"P",{"data-svelte-h":!0}),i(w)!=="svelte-119kr4u"&&(w.innerHTML=Ul),X=s(l),m=U(l,"P",{"data-svelte-h":!0}),i(m)!=="svelte-v5rsqz"&&(m.textContent=il),Y=s(l),p=U(l,"P",{"data-svelte-h":!0}),i(p)!=="svelte-vrcxc3"&&(p.textContent=ol),E=s(l),c=U(l,"OL",{"data-svelte-h":!0}),i(c)!=="svelte-t7gcm7"&&(c.innerHTML=jl),_=s(l),W(h.$$.fragment,l),V=s(l),j=U(l,"OL",{start:!0,"data-svelte-h":!0}),i(j)!=="svelte-17788hx"&&(j.innerHTML=Jl),$=s(l),W(I.$$.fragment,l),x=s(l),d=U(l,"P",{"data-svelte-h":!0}),i(d)!=="svelte-ghahgh"&&(d.innerHTML=ul),L=s(l),A=U(l,"P",{"data-svelte-h":!0}),i(A)!=="svelte-1h6gb73"&&(A.innerHTML=Tl),P=s(l),b=U(l,"P",{"data-svelte-h":!0}),i(b)!=="svelte-1732p7f"&&(b.innerHTML=Cl),D=s(l),g=U(l,"P",{"data-svelte-h":!0}),i(g)!=="svelte-1vz5lzw"&&(g.textContent=rl),O=s(l),f=U(l,"OL",{"data-svelte-h":!0}),i(f)!=="svelte-xngwau"&&(f.innerHTML=wl),K=s(l),W(G.$$.fragment,l),ll=s(l),J=U(l,"OL",{start:!0,"data-svelte-h":!0}),i(J)!=="svelte-cwcyc7"&&(J.innerHTML=ml),tl=s(l),W(u.$$.fragment,l),Ml=s(l),Z=U(l,"P",{"data-svelte-h":!0}),i(Z)!=="svelte-24o9mw"&&(Z.textContent=pl),el=s(l),W(v.$$.fragment,l),al=s(l),N=U(l,"P",{}),hl(N).forEach(M),this.h()},h(){nl(n,"name","hf:doc:metadata"),nl(n,"content",Ql),nl(j,"start","3"),nl(J,"start","4")},m(l,t){Zl(document.head,n),e(l,T,t),e(l,o,t),e(l,q,t),k(C,l,t),e(l,R,t),k(r,l,t),e(l,H,t),e(l,w,t),e(l,X,t),e(l,m,t),e(l,Y,t),e(l,p,t),e(l,E,t),e(l,c,t),e(l,_,t),k(h,l,t),e(l,V,t),e(l,j,t),e(l,$,t),k(I,l,t),e(l,x,t),e(l,d,t),e(l,L,t),e(l,A,t),e(l,P,t),e(l,b,t),e(l,D,t),e(l,g,t),e(l,O,t),e(l,f,t),e(l,K,t),k(G,l,t),e(l,ll,t),e(l,J,t),e(l,tl,t),k(u,l,t),e(l,Ml,t),e(l,Z,t),e(l,el,t),k(v,l,t),e(l,al,t),e(l,N,t),sl=!0},p(l,[t]){const cl={};t&2&&(cl.$$scope={dirty:t,ctx:l}),u.$set(cl)},i(l){sl||(Q(C.$$.fragment,l),Q(r.$$.fragment,l),Q(h.$$.fragment,l),Q(I.$$.fragment,l),Q(G.$$.fragment,l),Q(u.$$.fragment,l),Q(v.$$.fragment,l),sl=!0)},o(l){S(C.$$.fragment,l),S(r.$$.fragment,l),S(h.$$.fragment,l),S(I.$$.fragment,l),S(G.$$.fragment,l),S(u.$$.fragment,l),S(v.$$.fragment,l),sl=!1},d(l){l&&(M(T),M(o),M(q),M(R),M(H),M(w),M(X),M(m),M(Y),M(p),M(E),M(c),M(_),M(V),M(j),M($),M(x),M(d),M(L),M(A),M(P),M(b),M(D),M(g),M(O),M(f),M(K),M(ll),M(J),M(tl),M(Ml),M(Z),M(el),M(al),M(N)),M(n),F(C,l),F(r,l),F(h,l),F(I,l),F(G,l),F(u,l),F(v,l)}}}const Ql='{"title":"Contributing to multilingual evaluations","local":"contributing-to-multilingual-evaluations","sections":[{"title":"Contributing a small translation","local":"contributing-a-small-translation","sections":[],"depth":2},{"title":"Contributing a new multilingual task","local":"contributing-a-new-multilingual-task","sections":[],"depth":2}],"depth":1}';function Sl(z){return Al(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Hl extends gl{constructor(n){super(),fl(this,n,Sl,kl,dl,{})}}export{Hl as component};