lighteval/pr_1027/en/_app/immutable/nodes/18.1365c19b.js

import{s as $n,o as bn,n as _a}from"../chunks/scheduler.5f3e6389.js";import{S as yn,i as kn,e as n,s as a,c as p,h as xn,a as r,d as l,b as s,f as b,g,j as h,k as y,l as e,m as $,n as c,t as m,o as d,p as u}from"../chunks/index.373ab25c.js";import{C as Tn,H as nt,E as wn}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.c2e0d06d.js";import{D as k}from"../chunks/Docstring.bcc22899.js";import{C as $a}from"../chunks/CodeBlock.cd35d790.js";import{E as va}from"../chunks/ExampleCodeBlock.a41747f4.js";function Mn(j){let f,x;return f=new $a({props:{code:"ZG9jJTIwJTNEJTIwRG9jKCUwQSUyMCUyMCUyMCUyMHF1ZXJ5JTNEJTIyV2hhdCUyMGlzJTIwdGhlJTIwY2FwaXRhbCUyMG9mJTIwRnJhbmNlJTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwY2hvaWNlcyUzRCU1QiUyMkxvbmRvbiUyMiUyQyUyMCUyMlBhcmlzJTIyJTJDJTIwJTIyQmVybGluJTIyJTJDJTIwJTIyTWFkcmlkJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDElMkMlMjAlMjAlMjMlMjBQYXJpcyUyMGlzJTIwdGhlJTIwY29ycmVjdCUyMGFuc3dlciUwQSUyMCUyMCUyMCUyMGluc3RydWN0aW9uJTNEJTIyQW5zd2VyJTIwdGhlJTIwZm9sbG93aW5nJTIwZ2VvZ3JhcGh5JTIwcXVlc3Rpb24lM0ElMjIlMkMlMEEp",highlighted:`doc = Doc(
    query=<span class="hljs-string">&quot;What is the capital of France?&quot;</span>,
    choices=[<span class="hljs-string">&quot;London&quot;</span>, <span class="hljs-string">&quot;Paris&quot;</span>, <span class="hljs-string">&quot;Berlin&quot;</span>, <span class="hljs-string">&quot;Madrid&quot;</span>],
    gold_index=<span class="hljs-number">1</span>,  <span class="hljs-comment"># Paris is the correct answer</span>
    instruction=<span class="hljs-string">&quot;Answer the following geography question:&quot;</span>,
)`,wrap:!1}}),{c(){p(f.$$.fragment)},l(i){g(f.$$.fragment,i)},m(i,D){c(f,i,D),x=!0},p:_a,i(i){x||(m(f.$$.fragment,i),x=!0)},o(i){d(f.$$.fragment,i),x=!1},d(i){u(f,i)}}}function Jn(j){let f,x;return f=new $a({props:{code:"ZG9jJTIwJTNEJTIwRG9jKCUwQSUyMCUyMCUyMCUyMHF1ZXJ5JTNEJTIyV3JpdGUlMjBhJTIwc2hvcnQlMjBzdG9yeSUyMGFib3V0JTIwYSUyMHJvYm90LiUyMiUyQyUwQSUyMCUyMCUyMCUyMGNob2ljZXMlM0QlNUIlNUQlMkMlMjAlMjAlMjMlMjBObyUyMHByZWRlZmluZWQlMjBjaG9pY2VzJTIwZm9yJTIwZ2VuZXJhdGl2ZSUyMHRhc2tzJTBBJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDAlMkMlMjAlMjAlMjMlMjBOb3QlMjB1c2VkJTIwZm9yJTIwZ2VuZXJhdGl2ZSUyMHRhc2tzJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdGlvbl9zaXplJTNEMTAwJTJDJTBBJTIwJTIwJTIwJTIwc3RvcF9zZXF1ZW5jZXMlM0QlNUIlMjIlMEElMEFFbmQlMjIlNUQlMkMlMEEp",highlighted:`doc = Doc(
    query=<span class="hljs-string">&quot;Write a short story about a robot.&quot;</span>,
    choices=[],  <span class="hljs-comment"># No predefined choices for generative tasks</span>
    gold_index=<span class="hljs-number">0</span>,  <span class="hljs-comment"># Not used for generative tasks</span>
    generation_size=<span class="hljs-number">100</span>,
    stop_sequences=[<span class="hljs-string">&quot;

End&quot;</span>],
)`,wrap:!1}}),{c(){p(f.$$.fragment)},l(i){g(f.$$.fragment,i)},m(i,D){c(f,i,D),x=!0},p:_a,i(i){x||(m(f.$$.fragment,i),x=!0)},o(i){d(f.$$.fragment,i),x=!1},d(i){u(f,i)}}}function Cn(j){let f,x;return f=new $a({props:{code:"ZG9jJTIwJTNEJTIwRG9jKCUwQSUyMCUyMCUyMCUyMHF1ZXJ5JTNEJTIyVHJhbnNsYXRlJTIwJ0hlbGxvJTIwd29ybGQnJTIwdG8lMjBTcGFuaXNoLiUyMiUyQyUwQSUyMCUyMCUyMCUyMGNob2ljZXMlM0QlNUIlMjJIb2xhJTIwbXVuZG8lMjIlMkMlMjAlMjJCb25qb3VyJTIwbW9uZGUlMjIlMkMlMjAlMjJDaWFvJTIwbW9uZG8lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBnb2xkX2luZGV4JTNEMCUyQyUwQSUyMCUyMCUyMCUyMGZld3Nob3Rfc2FtcGxlcyUzRCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMERvYyhxdWVyeSUzRCUyMlRyYW5zbGF0ZSUyMCdHb29kJTIwbW9ybmluZyclMjB0byUyMFNwYW5pc2guJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2hvaWNlcyUzRCU1QiUyMkJ1ZW5vcyUyMGQlQzMlQURhcyUyMiUyQyUyMCUyMkJvbmpvdXIlMjIlMkMlMjAlMjJCdW9uZ2lvcm5vJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDApJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwRG9jKHF1ZXJ5JTNEJTIyVHJhbnNsYXRlJTIwJ1RoYW5rJTIweW91JyUyMHRvJTIwU3BhbmlzaC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjaG9pY2VzJTNEJTVCJTIyR3JhY2lhcyUyMiUyQyUyMCUyMk1lcmNpJTIyJTJDJTIwJTIyR3JhemllJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDApJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBKQ==",highlighted:`doc = Doc(
    query=<span class="hljs-string">&quot;Translate &#x27;Hello world&#x27; to Spanish.&quot;</span>,
    choices=[<span class="hljs-string">&quot;Hola mundo&quot;</span>, <span class="hljs-string">&quot;Bonjour monde&quot;</span>, <span class="hljs-string">&quot;Ciao mondo&quot;</span>],
    gold_index=<span class="hljs-number">0</span>,
    fewshot_samples=[
        Doc(query=<span class="hljs-string">&quot;Translate &#x27;Good morning&#x27; to Spanish.&quot;</span>,
            choices=[<span class="hljs-string">&quot;Buenos días&quot;</span>, <span class="hljs-string">&quot;Bonjour&quot;</span>, <span class="hljs-string">&quot;Buongiorno&quot;</span>],
            gold_index=<span class="hljs-number">0</span>),
        Doc(query=<span class="hljs-string">&quot;Translate &#x27;Thank you&#x27; to Spanish.&quot;</span>,
            choices=[<span class="hljs-string">&quot;Gracias&quot;</span>, <span class="hljs-string">&quot;Merci&quot;</span>, <span class="hljs-string">&quot;Grazie&quot;</span>],
            gold_index=<span class="hljs-number">0</span>)
    ],
)`,wrap:!1}}),{c(){p(f.$$.fragment)},l(i){g(f.$$.fragment,i)},m(i,D){c(f,i,D),x=!0},p:_a,i(i){x||(m(f.$$.fragment,i),x=!0)},o(i){d(f.$$.fragment,i),x=!1},d(i){u(f,i)}}}function In(j){let f,x;return f=new $a({props:{code:"ZG9jJTIwJTNEJTIwRG9jKCUwQSUyMCUyMCUyMCUyMHF1ZXJ5JTNEJTIyV2hhdCUyMGlzJTIwc2hvd24lMjBpbiUyMHRoaXMlMjBpbWFnZSUzRiUyMiUyQyUwQSUyMCUyMCUyMCUyMGNob2ljZXMlM0QlNUIlMjJBJTIwY2F0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDAlMkMlMEElMjAlMjAlMjAlMjBpbWFnZXMlM0QlNUJwaWxfaW1hZ2UlNUQlMkMlMjAlMjAlMjMlMjBQSUwlMjBJbWFnZSUyMG9iamVjdCUwQSk=",highlighted:`doc = Doc(
    query=<span class="hljs-string">&quot;What is shown in this image?&quot;</span>,
    choices=[<span class="hljs-string">&quot;A cat&quot;</span>],
    gold_index=<span class="hljs-number">0</span>,
    images=[pil_image],  <span class="hljs-comment"># PIL Image object</span>
)`,wrap:!1}}),{c(){p(f.$$.fragment)},l(i){g(f.$$.fragment,i)},m(i,D){c(f,i,D),x=!0},p:_a,i(i){x||(m(f.$$.fragment,i),x=!0)},o(i){d(f.$$.fragment,i),x=!1},d(i){u(f,i)}}}function Dn(j){let f,x,i,D,rt,Ee,lt,Pe,ot,Ge,it,Re,T,pt,ba,At,Cs="Configuration dataclass for a LightevalTask.",ya,Qt,Is=`This class stores all the configuration parameters needed to define and run
an evaluation task, including dataset information, prompt formatting,
evaluation metrics, and generation parameters.`,ka,zt,Ds=`Dataset Configuration:
hf_revision (str | None, optional): Specific dataset revision to use.
Defaults to None (latest).
hf_filter (Callable[[dict], bool] | None, optional): Filter function to
apply to dataset items. Defaults to None.
hf_avail_splits (ListLike[str], optional): Available dataset splits.
Defaults to [“train”, “validation”, “test”].`,xa,Ht,Us=`Evaluation Splits:
evaluation_splits (ListLike[str], optional): Dataset splits to use for
evaluation. Defaults to [“validation”].
few_shots_split (str | None, optional): Split to sample few-shot examples
from. Defaults to None.
few_shots_select (str | None, optional): Method for selecting few-shot
examples. Defaults to None.`,Ta,St,qs=`Generation Parameters:
generation_size (int | None, optional): Maximum token length for generated
text. Defaults to None.
generation_grammar (TextGenerationInputGrammarType | None, optional): Grammar
for structured text generation. Only available for TGI and Inference
Endpoint models. Defaults to None.
stop_sequence (ListLike[str] | None, optional): Sequences that stop text
generation. Defaults to None.
num_samples (list[int] | None, optional): Number of samples to generate
per input. Defaults to None.`,wa,Wt,js=`Task Configuration:
suite (ListLike[str], optional): Evaluation suites this task belongs to.
Defaults to [“custom”].
version (int, optional): Task version number. Increment when dataset or
prompt changes. Defaults to 0.
num_fewshots (int, optional): Number of few-shot examples to include.
Defaults to 0.
truncate_fewshots (bool, optional): Whether to truncate few-shot examples.
Defaults to False.
must_remove_duplicate_docs (bool, optional): Whether to remove duplicate
documents. Defaults to False.`,Ma,Xt,Ns=`Document Tracking:
original_num_docs (int, optional): Total number of documents in the task.
Defaults to -1.
effective_num_docs (int, optional): Number of documents actually used
in evaluation. Defaults to -1.`,Be,gt,Ve,w,ct,Ja,V,mt,Ca,Yt,Ls=`Return a dict with metric name and its aggregation function for all
metrics`,Ia,N,dt,Da,Ot,Es="Worker function to download a dataset from the HuggingFace Hub.",Ua,Kt,Ps=`Downloads the dataset specified in the task configuration, optionally
applies a filter if configured, and returns the dataset dictionary.
This method is designed to be used for parallel dataset loading.`,qa,Z,ut,ja,te,Gs="Returns the evaluation documents.",Na,F,ht,La,ee,Rs=`Returns the few shot documents. If the few shot documents are not
available, it gets them from the few shot split or the evaluation split.`,Ea,L,ft,Pa,ae,Bs="Get evaluation documents with few-shot examples and generation parameters configured.",Ga,se,Vs=`Retrieves evaluation documents, optionally limits the number of samples,
shuffles them for reproducibility, and configures each document with
few-shot examples and generation parameters for evaluation.`,Ra,A,vt,Ba,ne,Zs=`Parses the possible fewshot split keys in order: train, then validation
keys and matches them with the available keys.  Returns the first
available.`,Va,Q,_t,Za,re,Fs="Load datasets from the HuggingFace Hub for the given tasks.",Ze,$t,Fe,U,bt,Fa,z,yt,Aa,le,As="Prepare a prompt from a document, either using chat template or plain text format.",Qa,H,kt,za,oe,Qs=`Prepare a prompt for API calls, using a chat-like format.
Will not tokenize the message because APIs will usually handle this.`,Ae,xt,Qe,C,Tt,Ha,ie,zs="The Registry class is used to manage the task registry and get task classes.",Sa,S,wt,Wa,pe,Hs="Creates a custom task module to load tasks defined by the user in their own file.",Xa,W,Mt,Ya,ge,Ss="Create configuration tasks based on the provided meta_table.",Oa,X,Jt,Ka,ce,Ws="Print all the tasks in the task registry.",ze,Ct,He,v,It,ts,me,Xs="Dataclass representing a single evaluation sample for a benchmark.",es,de,Ys=`This class encapsulates all the information needed to evaluate a model on a single
task instance. It contains the input query, expected outputs, metadata, and
configuration parameters for different types of evaluation tasks.`,as,ue,Os="<strong>Required Fields:</strong>",ss,he,Ks="<li><code>query</code>: The input prompt or question</li> <li><code>choices</code>: Available answer choices (for multiple choice tasks)</li> <li><code>gold_index</code>: Index(es) of the correct answer(s)</li>",ns,fe,tn="<strong>Optional Fields:</strong>",rs,ve,en="<li><code>instruction</code>: System prompt, task specific. Will be appended to model specific system prompt.</li> <li><code>images</code>: Visual inputs for multimodal tasks.</li>",ls,_e,an=`Methods:
get_golds():
Returns the correct answer(s) as strings based on gold_index.
Handles both single and multiple correct answers.`,os,$e,sn="Usage Examples:",is,be,nn="<strong>Multiple Choice Question:</strong>",ps,Y,gs,ye,rn="<strong>Generative Task:</strong>",cs,O,ms,ke,ln="<strong>Few-shot Learning:</strong>",ds,K,us,xe,on="<strong>Multimodal Task:</strong>",hs,tt,fs,et,Dt,vs,Te,pn="Return gold targets extracted from the target dict",Se,Ut,We,q,qt,_s,at,jt,$s,we,gn="Get the original order of the data.",bs,st,Nt,ys,Me,cn="Iterator that yields the dataset splits based on the split limits.",Xe,Lt,Et,Ye,G,Pt,ks,I,Gt,xs,Je,mn=`Initialises the split limits based on generation parameters.
The splits are used to estimate time remaining when evaluating, and in the case of generative evaluations, to group similar samples together.`,Ts,Ce,dn="For generative tasks, self._sorting_criteria outputs:",ws,Ie,un="<li>a boolean (whether the generation task uses logits)</li> <li>a list (the stop sequences)</li> <li>the item length (the actual size sorting factor).</li>",Ms,De,hn=`In the current function, we create evaluation groups by generation parameters (logits and eos), so that samples with similar properties get batched together afterwards.
The samples will then be further organised by length in each split.`,Oe,Rt,Bt,Ke,R,Vt,Js,Ue,fn=`A distributed sampler that copy the last element only when drop_last is False so we keep a small padding in the batches
as our samples are sorted by length.`,ta,Zt,ea,Le,aa;return rt=new Tn({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),lt=new nt({props:{title:"Tasks",local:"tasks",headingTag:"h1"}}),ot=new nt({props:{title:"LightevalTask",local:"lightevaltask",headingTag:"h2"}}),it=new nt({props:{title:"LightevalTaskConfig",local:"lighteval.tasks.lighteval_task.LightevalTaskConfig",headingTag:"h3"}}),pt=new k({props:{name:"class lighteval.tasks.lighteval_task.LightevalTaskConfig",anchor:"lighteval.tasks.lighteval_task.LightevalTaskConfig",parameters:[{name:"name",val:": str"},{name:"prompt_function",val:": typing.Callable[[dict, str], lighteval.tasks.requests.Doc]"},{name:"hf_repo",val:": str"},{name:"hf_subset",val:": str"},{name:"metrics",val:": list[lighteval.metrics.utils.metric_utils.Metric] | tuple[lighteval.metrics.utils.metric_utils.Metric, ...]"},{name:"hf_revision",val:": str | None = None"},{name:"hf_filter",val:": typing.Optional[typing.Callable[[dict], bool]] = None"},{name:"hf_avail_splits",val:": list[str] | tuple[str, ...] = <factory>"},{name:"evaluation_splits",val:": list[str] | tuple[str, ...] = <factory>"},{name:"few_shots_split",val:": str | None = None"},{name:"few_shots_select",val:": str | None = None"},{name:"generation_size",val:": int | None = None"},{name:"generation_grammar",val:": huggingface_hub.inference._generated.types.text_generation.TextGenerationInputGrammarType | None = None"},{name:"stop_sequence",val:": list[str] | tuple[str, ...] | None = None"},{name:"num_samples",val:": list[int] | None = None"},{name:"suite",val:": list[str] | tuple[str, ...] = <factory>"},{name:"original_num_docs",val:": int = -1"},{name:"effective_num_docs",val:": int = -1"},{name:"must_remove_duplicate_docs",val:": bool = False"},{name:"num_fewshots",val:": int = 0"},{name:"version",val:": int = 0"}],parametersDescription:[{anchor:"lighteval.tasks.lighteval_task.LightevalTaskConfig.name",description:"<strong>name</strong> (str) &#x2014; Short name of the evaluation task.",name:"name"},{anchor:"lighteval.tasks.lighteval_task.LightevalTaskConfig.prompt_function",description:`<strong>prompt_function</strong> (Callable[[dict, str], Doc]) &#x2014; Function that converts dataset
row to Doc objects for evaluation. Takes a dataset row dict and task
name as input.`,name:"prompt_function"},{anchor:"lighteval.tasks.lighteval_task.LightevalTaskConfig.hf_repo",description:"<strong>hf_repo</strong> (str) &#x2014; HuggingFace Hub repository path containing the evaluation dataset.",name:"hf_repo"},{anchor:"lighteval.tasks.lighteval_task.LightevalTaskConfig.hf_subset",description:"<strong>hf_subset</strong> (str) &#x2014; Dataset subset/configuration name to use for this task.",name:"hf_subset"},{anchor:"lighteval.tasks.lighteval_task.LightevalTaskConfig.metrics",description:"<strong>metrics</strong> (ListLike[Metric]) &#x2014; List of metrics to compute for this task.",name:"metrics"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/lighteval_task.py#L47"}}),gt=new nt({props:{title:"LightevalTask",local:"lighteval.tasks.lighteval_task.LightevalTask",headingTag:"h3"}}),ct=new k({props:{name:"class lighteval.tasks.lighteval_task.LightevalTask",anchor:"lighteval.tasks.lighteval_task.LightevalTask",parameters:[{name:"config",val:": LightevalTaskConfig"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/lighteval_task.py#L193"}}),mt=new k({props:{name:"aggregation",anchor:"lighteval.tasks.lighteval_task.LightevalTask.aggregation",parameters:[],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/lighteval_task.py#L397"}}),dt=new k({props:{name:"download_dataset_worker",anchor:"lighteval.tasks.lighteval_task.LightevalTask.download_dataset_worker",parameters:[{name:"task",val:": LightevalTask"}],parametersDescription:[{anchor:"lighteval.tasks.lighteval_task.LightevalTask.download_dataset_worker.task",description:"<strong>task</strong> (LightevalTask) &#x2014; The task object containing dataset configuration.",name:"task"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/lighteval_task.py#L429",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The loaded dataset dictionary containing all splits.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>DatasetDict</p>
`}}),ut=new k({props:{name:"eval_docs",anchor:"lighteval.tasks.lighteval_task.LightevalTask.eval_docs",parameters:[],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/lighteval_task.py#L340",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Evaluation documents.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[Doc]</p>
`}}),ht=new k({props:{name:"fewshot_docs",anchor:"lighteval.tasks.lighteval_task.LightevalTask.fewshot_docs",parameters:[],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/lighteval_task.py#L321",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Documents that will be used for few shot examples. One
document = one few shot example.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[Doc]</p>
`}}),ft=new k({props:{name:"get_docs",anchor:"lighteval.tasks.lighteval_task.LightevalTask.get_docs",parameters:[{name:"max_samples",val:": int | None = None"}],parametersDescription:[{anchor:"lighteval.tasks.lighteval_task.LightevalTask.get_docs.max_samples",description:`<strong>max_samples</strong> (int | None, optional) &#x2014; Maximum number of documents to return.
If None, returns all available documents. Defaults to None.`,name:"max_samples"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/lighteval_task.py#L352",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of documents ready for evaluation with few-shot examples
and generation parameters configured.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[Doc]</p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>ValueError</code> — If no documents are available for evaluation.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>ValueError</code></p>
`}}),vt=new k({props:{name:"get_first_possible_fewshot_splits",anchor:"lighteval.tasks.lighteval_task.LightevalTask.get_first_possible_fewshot_splits",parameters:[{name:"available_splits",val:": list[str] | tuple[str, ...]"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/lighteval_task.py#L247",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>the first available fewshot splits or None if nothing is available</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>str</p>
`}}),_t=new k({props:{name:"load_datasets",anchor:"lighteval.tasks.lighteval_task.LightevalTask.load_datasets",parameters:[{name:"tasks",val:": dict"},{name:"dataset_loading_processes",val:": int = 1"}],parametersDescription:[{anchor:"lighteval.tasks.lighteval_task.LightevalTask.load_datasets.tasks",description:"<strong>tasks</strong> (dict[str, LightevalTask]) &#x2014; Dictionary mapping task names to task objects.",name:"tasks"},{anchor:"lighteval.tasks.lighteval_task.LightevalTask.load_datasets.dataset_loading_processes",description:`<strong>dataset_loading_processes</strong> (int, optional) &#x2014; Number of processes to use for
parallel dataset loading. Defaults to 1 (sequential loading).`,name:"dataset_loading_processes"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/lighteval_task.py#L406"}}),$t=new nt({props:{title:"PromptManager",local:"lighteval.tasks.prompt_manager.PromptManager",headingTag:"h2"}}),bt=new k({props:{name:"class lighteval.tasks.prompt_manager.PromptManager",anchor:"lighteval.tasks.prompt_manager.PromptManager",parameters:[{name:"use_chat_template",val:": bool = False"},{name:"tokenizer",val:" = None"},{name:"system_prompt",val:": str | None = None"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/prompt_manager.py#L42"}}),yt=new k({props:{name:"prepare_prompt",anchor:"lighteval.tasks.prompt_manager.PromptManager.prepare_prompt",parameters:[{name:"doc",val:": Doc"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/prompt_manager.py#L48",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The formatted prompt string</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>str</p>
`}}),kt=new k({props:{name:"prepare_prompt_api",anchor:"lighteval.tasks.prompt_manager.PromptManager.prepare_prompt_api",parameters:[{name:"doc",val:": Doc"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/prompt_manager.py#L88",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of message dictionaries for API calls</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[dict[str, str]]</p>
`}}),xt=new nt({props:{title:"Registry",local:"lighteval.tasks.registry.Registry",headingTag:"h2"}}),Tt=new k({props:{name:"class lighteval.tasks.registry.Registry",anchor:"lighteval.tasks.registry.Registry",parameters:[{name:"tasks",val:": str | pathlib.Path | None = None"},{name:"custom_tasks",val:": str | pathlib.Path | module | None = None"},{name:"load_community",val:": bool = False"},{name:"load_extended",val:": bool = False"},{name:"load_multilingual",val:": bool = False"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/registry.py#L111"}}),wt=new k({props:{name:"create_custom_tasks_module",anchor:"lighteval.tasks.registry.Registry.create_custom_tasks_module",parameters:[{name:"custom_tasks",val:": str | pathlib.Path | module"}],parametersDescription:[{anchor:"lighteval.tasks.registry.Registry.create_custom_tasks_module.custom_tasks",description:"<strong>custom_tasks</strong> (Optional[Union[str, ModuleType]]) &#x2014; Path to the custom tasks file or name of a module to import containing custom tasks or the module itself",name:"custom_tasks"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/registry.py#L378",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The newly imported/created custom tasks modules</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>ModuleType</p>
`}}),Mt=new k({props:{name:"create_task_config_dict",anchor:"lighteval.tasks.registry.Registry.create_task_config_dict",parameters:[{name:"meta_table",val:": list[lighteval.tasks.lighteval_task.LightevalTaskConfig] | None = None"}],parametersDescription:[{anchor:"lighteval.tasks.registry.Registry.create_task_config_dict.meta_table",description:`<strong>meta_table</strong> &#x2014; meta_table containing tasks
configurations. If not provided, it will be loaded from TABLE_PATH.`,name:"meta_table"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/registry.py#L403",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A dictionary of task names mapped to their corresponding LightevalTaskConfig.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Dict[str, LightevalTaskConfig]</p>
`}}),Jt=new k({props:{name:"print_all_tasks",anchor:"lighteval.tasks.registry.Registry.print_all_tasks",parameters:[{name:"suites",val:": str | None = None"}],parametersDescription:[{anchor:"lighteval.tasks.registry.Registry.print_all_tasks.suites",description:`<strong>suites</strong> &#x2014; Comma-separated list of suites to display. If None, shows core suites only.
Use &#x2018;all&#x2019; to show all available suites (core + optional).
Special handling for &#x2018;multilingual&#x2019; suite with dependency checking.`,name:"suites"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/registry.py#L425"}}),Ct=new nt({props:{title:"Doc",local:"lighteval.tasks.requests.Doc",headingTag:"h2"}}),It=new k({props:{name:"class lighteval.tasks.requests.Doc",anchor:"lighteval.tasks.requests.Doc",parameters:[{name:"query",val:": str"},{name:"choices",val:": list"},{name:"gold_index",val:": typing.Union[int, list[int]]"},{name:"instruction",val:": str | None = None"},{name:"images",val:": list['Image'] | None = None"},{name:"specific",val:": dict | None = None"},{name:"unconditioned_query",val:": str | None = None"},{name:"original_query",val:": str | None = None"},{name:"id",val:": str = ''"},{name:"task_name",val:": str = ''"},{name:"fewshot_samples",val:": list = <factory>"},{name:"sampling_methods",val:": list = <factory>"},{name:"fewshot_sorting_class",val:": str | None = None"},{name:"generation_size",val:": int | None = None"},{name:"stop_sequences",val:": list[str] | None = None"},{name:"use_logits",val:": bool = False"},{name:"num_samples",val:": int = 1"},{name:"generation_grammar",val:": None = None"}],parametersDescription:[{anchor:"lighteval.tasks.requests.Doc.query",description:`<strong>query</strong> (str) &#x2014;
The main query, prompt, or question to be sent to the model.`,name:"query"},{anchor:"lighteval.tasks.requests.Doc.choices",description:`<strong>choices</strong> (list[str]) &#x2014;
List of possible answer choices for the query.
For multiple choice tasks, this contains all options (A, B, C, D, etc.).
For generative tasks, this may be empty or contain reference answers.`,name:"choices"},{anchor:"lighteval.tasks.requests.Doc.gold_index",description:`<strong>gold_index</strong> (Union[int, list[int]]) &#x2014;
Index or indices of the correct answer(s) in the choices list.
For single correct answers,(e.g., 0 for first choice).
For multiple correct answers, use a list (e.g., [0, 2] for first and third).`,name:"gold_index"},{anchor:"lighteval.tasks.requests.Doc.instruction",description:`<strong>instruction</strong> (str | None) &#x2014;
System prompt or task-specific instructions to guide the model.
This is typically prepended to the query to set context or behavior.`,name:"instruction"},{anchor:"lighteval.tasks.requests.Doc.images",description:`<strong>images</strong> (list[&#x201C;Image&#x201D;] | None) &#x2014;
List of PIL Image objects for multimodal tasks.`,name:"images"},{anchor:"lighteval.tasks.requests.Doc.specific",description:`<strong>specific</strong> (dict | None) &#x2014;
Task-specific information or metadata.
Can contain any additional data needed for evaluation.`,name:"specific"},{anchor:"lighteval.tasks.requests.Doc.unconditioned_query",description:`<strong>unconditioned_query</strong> (Optional[str]) &#x2014;
Query without task-specific context for PMI normalization.
Used to calculate: log P(choice | Query) - log P(choice | Unconditioned Query).`,name:"unconditioned_query"},{anchor:"lighteval.tasks.requests.Doc.original_query",description:`<strong>original_query</strong> (str | None) &#x2014;
The query before any preprocessing or modification.`,name:"original_query"},{anchor:"lighteval.tasks.requests.Doc.#",description:"<strong>#</strong> Set by task parameters &#x2014;",name:"#"},{anchor:"lighteval.tasks.requests.Doc.id",description:`<strong>id</strong> (str) &#x2014;
Unique identifier for this evaluation instance.
Set by the task and not the user.`,name:"id"},{anchor:"lighteval.tasks.requests.Doc.task_name",description:`<strong>task_name</strong> (str) &#x2014;
Name of the task or benchmark this Doc belongs to.`,name:"task_name"},{anchor:"lighteval.tasks.requests.Doc.##",description:"<strong>##</strong> Few-shot Learning Parameters &#x2014;",name:"##"},{anchor:"lighteval.tasks.requests.Doc.fewshot_samples",description:`<strong>fewshot_samples</strong> (list) &#x2014;
List of Doc objects representing few-shot examples.
These examples are prepended to the main query to provide context.`,name:"fewshot_samples"},{anchor:"lighteval.tasks.requests.Doc.sampling_methods",description:`<strong>sampling_methods</strong> (list[SamplingMethod]) &#x2014;
List of sampling methods to use for this instance.
Options: GENERATIVE, LOGPROBS, PERPLEXITY.`,name:"sampling_methods"},{anchor:"lighteval.tasks.requests.Doc.fewshot_sorting_class",description:`<strong>fewshot_sorting_class</strong> (Optional[str]) &#x2014;
Class label for balanced few-shot example selection.
Used to ensure diverse representation in few-shot examples.`,name:"fewshot_sorting_class"},{anchor:"lighteval.tasks.requests.Doc.##",description:"<strong>##</strong> Generation Control Parameters &#x2014;",name:"##"},{anchor:"lighteval.tasks.requests.Doc.generation_size",description:`<strong>generation_size</strong> (int | None) &#x2014;
Maximum number of tokens to generate for this instance.`,name:"generation_size"},{anchor:"lighteval.tasks.requests.Doc.stop_sequences",description:`<strong>stop_sequences</strong> (list[str] | None) &#x2014;
List of strings that should stop generation when encountered.
<strong>Used for</strong>: Controlled generation, preventing unwanted continuations.`,name:"stop_sequences"},{anchor:"lighteval.tasks.requests.Doc.use_logits",description:`<strong>use_logits</strong> (bool) &#x2014;
Whether to return logits (raw model outputs) in addition to text.
<strong>Used for</strong>: Probability analysis, confidence scoring, detailed evaluation.`,name:"use_logits"},{anchor:"lighteval.tasks.requests.Doc.num_samples",description:`<strong>num_samples</strong> (int) &#x2014;
Number of different samples to generate for this instance.
<strong>Used for</strong>: Diversity analysis, uncertainty estimation, ensemble methods.`,name:"num_samples"},{anchor:"lighteval.tasks.requests.Doc.generation_grammar",description:`<strong>generation_grammar</strong> (None) &#x2014;
Grammar constraints for generation (currently not implemented).
<strong>Reserved for</strong>: Future structured generation features.`,name:"generation_grammar"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/requests.py#L44"}}),Y=new va({props:{anchor:"lighteval.tasks.requests.Doc.example",$$slots:{default:[Mn]},$$scope:{ctx:j}}}),O=new va({props:{anchor:"lighteval.tasks.requests.Doc.example-2",$$slots:{default:[Jn]},$$scope:{ctx:j}}}),K=new va({props:{anchor:"lighteval.tasks.requests.Doc.example-3",$$slots:{default:[Cn]},$$scope:{ctx:j}}}),tt=new va({props:{anchor:"lighteval.tasks.requests.Doc.example-4",$$slots:{default:[In]},$$scope:{ctx:j}}}),Dt=new k({props:{name:"get_golds",anchor:"lighteval.tasks.requests.Doc.get_golds",parameters:[],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/tasks/requests.py#L217"}}),Ut=new nt({props:{title:"Datasets",local:"lighteval.data.DynamicBatchDataset",headingTag:"h2"}}),qt=new k({props:{name:"class lighteval.data.DynamicBatchDataset",anchor:"lighteval.data.DynamicBatchDataset",parameters:[{name:"requests",val:": list"},{name:"num_dataset_splits",val:": int"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/data.py#L44"}}),jt=new k({props:{name:"get_original_order",anchor:"lighteval.data.DynamicBatchDataset.get_original_order",parameters:[{name:"new_arr",val:": list"}],parametersDescription:[{anchor:"lighteval.data.DynamicBatchDataset.get_original_order.new_arr",description:`<strong>new_arr</strong> (list) &#x2014; Array containing any kind of data that needs to be
reset in the original order.`,name:"new_arr"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/data.py#L88",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>new_arr in the original order.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list</p>
`}}),Nt=new k({props:{name:"splits_iterator",anchor:"lighteval.data.DynamicBatchDataset.splits_iterator",parameters:[],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/data.py#L110",returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Subset</p>
`,isYield:!0}}),Et=new k({props:{name:"class lighteval.data.LoglikelihoodDataset",anchor:"lighteval.data.LoglikelihoodDataset",parameters:[{name:"requests",val:": list"},{name:"num_dataset_splits",val:": int"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/data.py#L161"}}),Pt=new k({props:{name:"class lighteval.data.GenerativeTaskDataset",anchor:"lighteval.data.GenerativeTaskDataset",parameters:[{name:"requests",val:": list"},{name:"num_dataset_splits",val:": int"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/data.py#L186"}}),Gt=new k({props:{name:"init_split_limits",anchor:"lighteval.data.GenerativeTaskDataset.init_split_limits",parameters:[{name:"num_dataset_splits",val:""}],parametersDescription:[{anchor:"lighteval.data.GenerativeTaskDataset.init_split_limits.num_dataset_splits",description:"<strong>num_dataset_splits</strong> (<em>type</em>) &#x2014; <em>description</em>",name:"num_dataset_splits"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/data.py#L187",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><em>description</em></p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><em>type</em></p>
`}}),Bt=new k({props:{name:"class lighteval.data.GenerativeTaskDatasetNanotron",anchor:"lighteval.data.GenerativeTaskDatasetNanotron",parameters:[{name:"requests",val:": list"},{name:"num_dataset_splits",val:": int"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/data.py#L254"}}),Vt=new k({props:{name:"class lighteval.data.GenDistributedSampler",anchor:"lighteval.data.GenDistributedSampler",parameters:[{name:"dataset",val:": Dataset"},{name:"num_replicas",val:": typing.Optional[int] = None"},{name:"rank",val:": typing.Optional[int] = None"},{name:"shuffle",val:": bool = True"},{name:"seed",val:": int = 0"},{name:"drop_last",val:": bool = False"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/data.py#L270"}}),Zt=new wn({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/package_reference/tasks.mdx"}}),{c(){f=n("meta"),x=a(),i=n("p"),D=a(),p(rt.$$.fragment),Ee=a(),p(lt.$$.fragment),Pe=a(),p(ot.$$.fragment),Ge=a(),p(it.$$.fragment),Re=a(),T=n("div"),p(pt.$$.fragment),ba=a(),At=n("p"),At.textContent=Cs,ya=a(),Qt=n("p"),Qt.textContent=Is,ka=a(),zt=n("p"),zt.textContent=Ds,xa=a(),Ht=n("p"),Ht.textContent=Us,Ta=a(),St=n("p"),St.textContent=qs,wa=a(),Wt=n("p"),Wt.textContent=js,Ma=a(),Xt=n("p"),Xt.textContent=Ns,Be=a(),p(gt.$$.fragment),Ve=a(),w=n("div"),p(ct.$$.fragment),Ja=a(),V=n("div"),p(mt.$$.fragment),Ca=a(),Yt=n("p"),Yt.textContent=Ls,Ia=a(),N=n("div"),p(dt.$$.fragment),Da=a(),Ot=n("p"),Ot.textContent=Es,Ua=a(),Kt=n("p"),Kt.textContent=Ps,qa=a(),Z=n("div"),p(ut.$$.fragment),ja=a(),te=n("p"),te.textContent=Gs,Na=a(),F=n("div"),p(ht.$$.fragment),La=a(),ee=n("p"),ee.textContent=Rs,Ea=a(),L=n("div"),p(ft.$$.fragment),Pa=a(),ae=n("p"),ae.textContent=Bs,Ga=a(),se=n("p"),se.textContent=Vs,Ra=a(),A=n("div"),p(vt.$$.fragment),Ba=a(),ne=n("p"),ne.textContent=Zs,Va=a(),Q=n("div"),p(_t.$$.fragment),Za=a(),re=n("p"),re.textContent=Fs,Ze=a(),p($t.$$.fragment),Fe=a(),U=n("div"),p(bt.$$.fragment),Fa=a(),z=n("div"),p(yt.$$.fragment),Aa=a(),le=n("p"),le.textContent=As,Qa=a(),H=n("div"),p(kt.$$.fragment),za=a(),oe=n("p"),oe.textContent=Qs,Ae=a(),p(xt.$$.fragment),Qe=a(),C=n("div"),p(Tt.$$.fragment),Ha=a(),ie=n("p"),ie.textContent=zs,Sa=a(),S=n("div"),p(wt.$$.fragment),Wa=a(),pe=n("p"),pe.textContent=Hs,Xa=a(),W=n("div"),p(Mt.$$.fragment),Ya=a(),ge=n("p"),ge.textContent=Ss,Oa=a(),X=n("div"),p(Jt.$$.fragment),Ka=a(),ce=n("p"),ce.textContent=Ws,ze=a(),p(Ct.$$.fragment),He=a(),v=n("div"),p(It.$$.fragment),ts=a(),me=n("p"),me.textContent=Xs,es=a(),de=n("p"),de.textContent=Ys,as=a(),ue=n("p"),ue.innerHTML=Os,ss=a(),he=n("ul"),he.innerHTML=Ks,ns=a(),fe=n("p"),fe.innerHTML=tn,rs=a(),ve=n("ul"),ve.innerHTML=en,ls=a(),_e=n("p"),_e.textContent=an,os=a(),$e=n("p"),$e.textContent=sn,is=a(),be=n("p"),be.innerHTML=nn,ps=a(),p(Y.$$.fragment),gs=a(),ye=n("p"),ye.innerHTML=rn,cs=a(),p(O.$$.fragment),ms=a(),ke=n("p"),ke.innerHTML=ln,ds=a(),p(K.$$.fragment),us=a(),xe=n("p"),xe.innerHTML=on,hs=a(),p(tt.$$.fragment),fs=a(),et=n("div"),p(Dt.$$.fragment),vs=a(),Te=n("p"),Te.textContent=pn,Se=a(),p(Ut.$$.fragment),We=a(),q=n("div"),p(qt.$$.fragment),_s=a(),at=n("div"),p(jt.$$.fragment),$s=a(),we=n("p"),we.textContent=gn,bs=a(),st=n("div"),p(Nt.$$.fragment),ys=a(),Me=n("p"),Me.textContent=cn,Xe=a(),Lt=n("div"),p(Et.$$.fragment),Ye=a(),G=n("div"),p(Pt.$$.fragment),ks=a(),I=n("div"),p(Gt.$$.fragment),xs=a(),Je=n("p"),Je.textContent=mn,Ts=a(),Ce=n("p"),Ce.textContent=dn,ws=a(),Ie=n("ul"),Ie.innerHTML=un,Ms=a(),De=n("p"),De.textContent=hn,Oe=a(),Rt=n("div"),p(Bt.$$.fragment),Ke=a(),R=n("div"),p(Vt.$$.fragment),Js=a(),Ue=n("p"),Ue.textContent=fn,ta=a(),p(Zt.$$.fragment),ea=a(),Le=n("p"),this.h()},l(t){const o=xn("svelte-u9bgzb",document.head);f=r(o,"META",{name:!0,content:!0}),o.forEach(l),x=s(t),i=r(t,"P",{}),b(i).forEach(l),D=s(t),g(rt.$$.fragment,t),Ee=s(t),g(lt.$$.fragment,t),Pe=s(t),g(ot.$$.fragment,t),Ge=s(t),g(it.$$.fragment,t),Re=s(t),T=r(t,"DIV",{class:!0});var M=b(T);g(pt.$$.fragment,M),ba=s(M),At=r(M,"P",{"data-svelte-h":!0}),h(At)!=="svelte-stmfsf"&&(At.textContent=Cs),ya=s(M),Qt=r(M,"P",{"data-svelte-h":!0}),h(Qt)!=="svelte-1wprphn"&&(Qt.textContent=Is),ka=s(M),zt=r(M,"P",{"data-svelte-h":!0}),h(zt)!=="svelte-1y7wfzb"&&(zt.textContent=Ds),xa=s(M),Ht=r(M,"P",{"data-svelte-h":!0}),h(Ht)!=="svelte-p0lesm"&&(Ht.textContent=Us),Ta=s(M),St=r(M,"P",{"data-svelte-h":!0}),h(St)!=="svelte-18lh4ux"&&(St.textContent=qs),wa=s(M),Wt=r(M,"P",{"data-svelte-h":!0}),h(Wt)!=="svelte-ix8jj3"&&(Wt.textContent=js),Ma=s(M),Xt=r(M,"P",{"data-svelte-h":!0}),h(Xt)!=="svelte-lb2xyc"&&(Xt.textContent=Ns),M.forEach(l),Be=s(t),g(gt.$$.fragment,t),Ve=s(t),w=r(t,"DIV",{class:!0});var J=b(w);g(ct.$$.fragment,J),Ja=s(J),V=r(J,"DIV",{class:!0});var Ft=b(V);g(mt.$$.fragment,Ft),Ca=s(Ft),Yt=r(Ft,"P",{"data-svelte-h":!0}),h(Yt)!=="svelte-e3kmxq"&&(Yt.textContent=Ls),Ft.forEach(l),Ia=s(J),N=r(J,"DIV",{class:!0});var B=b(N);g(dt.$$.fragment,B),Da=s(B),Ot=r(B,"P",{"data-svelte-h":!0}),h(Ot)!=="svelte-vbykr6"&&(Ot.textContent=Es),Ua=s(B),Kt=r(B,"P",{"data-svelte-h":!0}),h(Kt)!=="svelte-1yjy9hy"&&(Kt.textContent=Ps),B.forEach(l),qa=s(J),Z=r(J,"DIV",{class:!0});var sa=b(Z);g(ut.$$.fragment,sa),ja=s(sa),te=r(sa,"P",{"data-svelte-h":!0}),h(te)!=="svelte-qytfqy"&&(te.textContent=Gs),sa.forEach(l),Na=s(J),F=r(J,"DIV",{class:!0});var na=b(F);g(ht.$$.fragment,na),La=s(na),ee=r(na,"P",{"data-svelte-h":!0}),h(ee)!=="svelte-1y4eieh"&&(ee.textContent=Rs),na.forEach(l),Ea=s(J),L=r(J,"DIV",{class:!0});var qe=b(L);g(ft.$$.fragment,qe),Pa=s(qe),ae=r(qe,"P",{"data-svelte-h":!0}),h(ae)!=="svelte-19ponm9"&&(ae.textContent=Bs),Ga=s(qe),se=r(qe,"P",{"data-svelte-h":!0}),h(se)!=="svelte-tler3x"&&(se.textContent=Vs),qe.forEach(l),Ra=s(J),A=r(J,"DIV",{class:!0});var ra=b(A);g(vt.$$.fragment,ra),Ba=s(ra),ne=r(ra,"P",{"data-svelte-h":!0}),h(ne)!=="svelte-1283lse"&&(ne.textContent=Zs),ra.forEach(l),Va=s(J),Q=r(J,"DIV",{class:!0});var la=b(Q);g(_t.$$.fragment,la),Za=s(la),re=r(la,"P",{"data-svelte-h":!0}),h(re)!=="svelte-sf66t8"&&(re.textContent=Fs),la.forEach(l),J.forEach(l),Ze=s(t),g($t.$$.fragment,t),Fe=s(t),U=r(t,"DIV",{class:!0});var je=b(U);g(bt.$$.fragment,je),Fa=s(je),z=r(je,"DIV",{class:!0});var oa=b(z);g(yt.$$.fragment,oa),Aa=s(oa),le=r(oa,"P",{"data-svelte-h":!0}),h(le)!=="svelte-1ttqjjc"&&(le.textContent=As),oa.forEach(l),Qa=s(je),H=r(je,"DIV",{class:!0});var ia=b(H);g(kt.$$.fragment,ia),za=s(ia),oe=r(ia,"P",{"data-svelte-h":!0}),h(oe)!=="svelte-ky83kg"&&(oe.textContent=Qs),ia.forEach(l),je.forEach(l),Ae=s(t),g(xt.$$.fragment,t),Qe=s(t),C=r(t,"DIV",{class:!0});var E=b(C);g(Tt.$$.fragment,E),Ha=s(E),ie=r(E,"P",{"data-svelte-h":!0}),h(ie)!=="svelte-2fh3bo"&&(ie.textContent=zs),Sa=s(E),S=r(E,"DIV",{class:!0});var pa=b(S);g(wt.$$.fragment,pa),Wa=s(pa),pe=r(pa,"P",{"data-svelte-h":!0}),h(pe)!=="svelte-m08mne"&&(pe.textContent=Hs),pa.forEach(l),Xa=s(E),W=r(E,"DIV",{class:!0});var ga=b(W);g(Mt.$$.fragment,ga),Ya=s(ga),ge=r(ga,"P",{"data-svelte-h":!0}),h(ge)!=="svelte-126hqbu"&&(ge.textContent=Ss),ga.forEach(l),Oa=s(E),X=r(E,"DIV",{class:!0});var ca=b(X);g(Jt.$$.fragment,ca),Ka=s(ca),ce=r(ca,"P",{"data-svelte-h":!0}),h(ce)!=="svelte-x1foin"&&(ce.textContent=Ws),ca.forEach(l),E.forEach(l),ze=s(t),g(Ct.$$.fragment,t),He=s(t),v=r(t,"DIV",{class:!0});var _=b(v);g(It.$$.fragment,_),ts=s(_),me=r(_,"P",{"data-svelte-h":!0}),h(me)!=="svelte-ekxass"&&(me.textContent=Xs),es=s(_),de=r(_,"P",{"data-svelte-h":!0}),h(de)!=="svelte-mg9ows"&&(de.textContent=Ys),as=s(_),ue=r(_,"P",{"data-svelte-h":!0}),h(ue)!=="svelte-tguus5"&&(ue.innerHTML=Os),ss=s(_),he=r(_,"UL",{"data-svelte-h":!0}),h(he)!=="svelte-upt8y4"&&(he.innerHTML=Ks),ns=s(_),fe=r(_,"P",{"data-svelte-h":!0}),h(fe)!=="svelte-1opiqf6"&&(fe.innerHTML=tn),rs=s(_),ve=r(_,"UL",{"data-svelte-h":!0}),h(ve)!=="svelte-1tfzood"&&(ve.innerHTML=en),ls=s(_),_e=r(_,"P",{"data-svelte-h":!0}),h(_e)!=="svelte-a0ub3g"&&(_e.textContent=an),os=s(_),$e=r(_,"P",{"data-svelte-h":!0}),h($e)!=="svelte-93h1jg"&&($e.textContent=sn),is=s(_),be=r(_,"P",{"data-svelte-h":!0}),h(be)!=="svelte-1tze8ns"&&(be.innerHTML=nn),ps=s(_),g(Y.$$.fragment,_),gs=s(_),ye=r(_,"P",{"data-svelte-h":!0}),h(ye)!=="svelte-ydqksw"&&(ye.innerHTML=rn),cs=s(_),g(O.$$.fragment,_),ms=s(_),ke=r(_,"P",{"data-svelte-h":!0}),h(ke)!=="svelte-t0g8ra"&&(ke.innerHTML=ln),ds=s(_),g(K.$$.fragment,_),us=s(_),xe=r(_,"P",{"data-svelte-h":!0}),h(xe)!=="svelte-fraz6o"&&(xe.innerHTML=on),hs=s(_),g(tt.$$.fragment,_),fs=s(_),et=r(_,"DIV",{class:!0});var ma=b(et);g(Dt.$$.fragment,ma),vs=s(ma),Te=r(ma,"P",{"data-svelte-h":!0}),h(Te)!=="svelte-motf0a"&&(Te.textContent=pn),ma.forEach(l),_.forEach(l),Se=s(t),g(Ut.$$.fragment,t),We=s(t),q=r(t,"DIV",{class:!0});var Ne=b(q);g(qt.$$.fragment,Ne),_s=s(Ne),at=r(Ne,"DIV",{class:!0});var da=b(at);g(jt.$$.fragment,da),$s=s(da),we=r(da,"P",{"data-svelte-h":!0}),h(we)!=="svelte-17n3dve"&&(we.textContent=gn),da.forEach(l),bs=s(Ne),st=r(Ne,"DIV",{class:!0});var ua=b(st);g(Nt.$$.fragment,ua),ys=s(ua),Me=r(ua,"P",{"data-svelte-h":!0}),h(Me)!=="svelte-udia6y"&&(Me.textContent=cn),ua.forEach(l),Ne.forEach(l),Xe=s(t),Lt=r(t,"DIV",{class:!0});var vn=b(Lt);g(Et.$$.fragment,vn),vn.forEach(l),Ye=s(t),G=r(t,"DIV",{class:!0});var ha=b(G);g(Pt.$$.fragment,ha),ks=s(ha),I=r(ha,"DIV",{class:!0});var P=b(I);g(Gt.$$.fragment,P),xs=s(P),Je=r(P,"P",{"data-svelte-h":!0}),h(Je)!=="svelte-1akk4jf"&&(Je.textContent=mn),Ts=s(P),Ce=r(P,"P",{"data-svelte-h":!0}),h(Ce)!=="svelte-1jovh48"&&(Ce.textContent=dn),ws=s(P),Ie=r(P,"UL",{"data-svelte-h":!0}),h(Ie)!=="svelte-62kt3p"&&(Ie.innerHTML=un),Ms=s(P),De=r(P,"P",{"data-svelte-h":!0}),h(De)!=="svelte-1oovzw"&&(De.textContent=hn),P.forEach(l),ha.forEach(l),Oe=s(t),Rt=r(t,"DIV",{class:!0});var _n=b(Rt);g(Bt.$$.fragment,_n),_n.forEach(l),Ke=s(t),R=r(t,"DIV",{class:!0});var fa=b(R);g(Vt.$$.fragment,fa),Js=s(fa),Ue=r(fa,"P",{"data-svelte-h":!0}),h(Ue)!=="svelte-1xq59t"&&(Ue.textContent=fn),fa.forEach(l),ta=s(t),g(Zt.$$.fragment,t),ea=s(t),Le=r(t,"P",{}),b(Le).forEach(l),this.h()},h(){y(f,"name","hf:doc:metadata"),y(f,"content",Un),y(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(et,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(at,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(st,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(Lt,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(Rt,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,o){e(document.head,f),$(t,x,o),$(t,i,o),$(t,D,o),c(rt,t,o),$(t,Ee,o),c(lt,t,o),$(t,Pe,o),c(ot,t,o),$(t,Ge,o),c(it,t,o),$(t,Re,o),$(t,T,o),c(pt,T,null),e(T,ba),e(T,At),e(T,ya),e(T,Qt),e(T,ka),e(T,zt),e(T,xa),e(T,Ht),e(T,Ta),e(T,St),e(T,wa),e(T,Wt),e(T,Ma),e(T,Xt),$(t,Be,o),c(gt,t,o),$(t,Ve,o),$(t,w,o),c(ct,w,null),e(w,Ja),e(w,V),c(mt,V,null),e(V,Ca),e(V,Yt),e(w,Ia),e(w,N),c(dt,N,null),e(N,Da),e(N,Ot),e(N,Ua),e(N,Kt),e(w,qa),e(w,Z),c(ut,Z,null),e(Z,ja),e(Z,te),e(w,Na),e(w,F),c(ht,F,null),e(F,La),e(F,ee),e(w,Ea),e(w,L),c(ft,L,null),e(L,Pa),e(L,ae),e(L,Ga),e(L,se),e(w,Ra),e(w,A),c(vt,A,null),e(A,Ba),e(A,ne),e(w,Va),e(w,Q),c(_t,Q,null),e(Q,Za),e(Q,re),$(t,Ze,o),c($t,t,o),$(t,Fe,o),$(t,U,o),c(bt,U,null),e(U,Fa),e(U,z),c(yt,z,null),e(z,Aa),e(z,le),e(U,Qa),e(U,H),c(kt,H,null),e(H,za),e(H,oe),$(t,Ae,o),c(xt,t,o),$(t,Qe,o),$(t,C,o),c(Tt,C,null),e(C,Ha),e(C,ie),e(C,Sa),e(C,S),c(wt,S,null),e(S,Wa),e(S,pe),e(C,Xa),e(C,W),c(Mt,W,null),e(W,Ya),e(W,ge),e(C,Oa),e(C,X),c(Jt,X,null),e(X,Ka),e(X,ce),$(t,ze,o),c(Ct,t,o),$(t,He,o),$(t,v,o),c(It,v,null),e(v,ts),e(v,me),e(v,es),e(v,de),e(v,as),e(v,ue),e(v,ss),e(v,he),e(v,ns),e(v,fe),e(v,rs),e(v,ve),e(v,ls),e(v,_e),e(v,os),e(v,$e),e(v,is),e(v,be),e(v,ps),c(Y,v,null),e(v,gs),e(v,ye),e(v,cs),c(O,v,null),e(v,ms),e(v,ke),e(v,ds),c(K,v,null),e(v,us),e(v,xe),e(v,hs),c(tt,v,null),e(v,fs),e(v,et),c(Dt,et,null),e(et,vs),e(et,Te),$(t,Se,o),c(Ut,t,o),$(t,We,o),$(t,q,o),c(qt,q,null),e(q,_s),e(q,at),c(jt,at,null),e(at,$s),e(at,we),e(q,bs),e(q,st),c(Nt,st,null),e(st,ys),e(st,Me),$(t,Xe,o),$(t,Lt,o),c(Et,Lt,null),$(t,Ye,o),$(t,G,o),c(Pt,G,null),e(G,ks),e(G,I),c(Gt,I,null),e(I,xs),e(I,Je),e(I,Ts),e(I,Ce),e(I,ws),e(I,Ie),e(I,Ms),e(I,De),$(t,Oe,o),$(t,Rt,o),c(Bt,Rt,null),$(t,Ke,o),$(t,R,o),c(Vt,R,null),e(R,Js),e(R,Ue),$(t,ta,o),c(Zt,t,o),$(t,ea,o),$(t,Le,o),aa=!0},p(t,[o]){const M={};o&2&&(M.$$scope={dirty:o,ctx:t}),Y.$set(M);const J={};o&2&&(J.$$scope={dirty:o,ctx:t}),O.$set(J);const Ft={};o&2&&(Ft.$$scope={dirty:o,ctx:t}),K.$set(Ft);const B={};o&2&&(B.$$scope={dirty:o,ctx:t}),tt.$set(B)},i(t){aa||(m(rt.$$.fragment,t),m(lt.$$.fragment,t),m(ot.$$.fragment,t),m(it.$$.fragment,t),m(pt.$$.fragment,t),m(gt.$$.fragment,t),m(ct.$$.fragment,t),m(mt.$$.fragment,t),m(dt.$$.fragment,t),m(ut.$$.fragment,t),m(ht.$$.fragment,t),m(ft.$$.fragment,t),m(vt.$$.fragment,t),m(_t.$$.fragment,t),m($t.$$.fragment,t),m(bt.$$.fragment,t),m(yt.$$.fragment,t),m(kt.$$.fragment,t),m(xt.$$.fragment,t),m(Tt.$$.fragment,t),m(wt.$$.fragment,t),m(Mt.$$.fragment,t),m(Jt.$$.fragment,t),m(Ct.$$.fragment,t),m(It.$$.fragment,t),m(Y.$$.fragment,t),m(O.$$.fragment,t),m(K.$$.fragment,t),m(tt.$$.fragment,t),m(Dt.$$.fragment,t),m(Ut.$$.fragment,t),m(qt.$$.fragment,t),m(jt.$$.fragment,t),m(Nt.$$.fragment,t),m(Et.$$.fragment,t),m(Pt.$$.fragment,t),m(Gt.$$.fragment,t),m(Bt.$$.fragment,t),m(Vt.$$.fragment,t),m(Zt.$$.fragment,t),aa=!0)},o(t){d(rt.$$.fragment,t),d(lt.$$.fragment,t),d(ot.$$.fragment,t),d(it.$$.fragment,t),d(pt.$$.fragment,t),d(gt.$$.fragment,t),d(ct.$$.fragment,t),d(mt.$$.fragment,t),d(dt.$$.fragment,t),d(ut.$$.fragment,t),d(ht.$$.fragment,t),d(ft.$$.fragment,t),d(vt.$$.fragment,t),d(_t.$$.fragment,t),d($t.$$.fragment,t),d(bt.$$.fragment,t),d(yt.$$.fragment,t),d(kt.$$.fragment,t),d(xt.$$.fragment,t),d(Tt.$$.fragment,t),d(wt.$$.fragment,t),d(Mt.$$.fragment,t),d(Jt.$$.fragment,t),d(Ct.$$.fragment,t),d(It.$$.fragment,t),d(Y.$$.fragment,t),d(O.$$.fragment,t),d(K.$$.fragment,t),d(tt.$$.fragment,t),d(Dt.$$.fragment,t),d(Ut.$$.fragment,t),d(qt.$$.fragment,t),d(jt.$$.fragment,t),d(Nt.$$.fragment,t),d(Et.$$.fragment,t),d(Pt.$$.fragment,t),d(Gt.$$.fragment,t),d(Bt.$$.fragment,t),d(Vt.$$.fragment,t),d(Zt.$$.fragment,t),aa=!1},d(t){t&&(l(x),l(i),l(D),l(Ee),l(Pe),l(Ge),l(Re),l(T),l(Be),l(Ve),l(w),l(Ze),l(Fe),l(U),l(Ae),l(Qe),l(C),l(ze),l(He),l(v),l(Se),l(We),l(q),l(Xe),l(Lt),l(Ye),l(G),l(Oe),l(Rt),l(Ke),l(R),l(ta),l(ea),l(Le)),l(f),u(rt,t),u(lt,t),u(ot,t),u(it,t),u(pt),u(gt,t),u(ct),u(mt),u(dt),u(ut),u(ht),u(ft),u(vt),u(_t),u($t,t),u(bt),u(yt),u(kt),u(xt,t),u(Tt),u(wt),u(Mt),u(Jt),u(Ct,t),u(It),u(Y),u(O),u(K),u(tt),u(Dt),u(Ut,t),u(qt),u(jt),u(Nt),u(Et),u(Pt),u(Gt),u(Bt),u(Vt),u(Zt,t)}}}const Un='{"title":"Tasks","local":"tasks","sections":[{"title":"LightevalTask","local":"lightevaltask","sections":[{"title":"LightevalTaskConfig","local":"lighteval.tasks.lighteval_task.LightevalTaskConfig","sections":[],"depth":3},{"title":"LightevalTask","local":"lighteval.tasks.lighteval_task.LightevalTask","sections":[],"depth":3}],"depth":2},{"title":"PromptManager","local":"lighteval.tasks.prompt_manager.PromptManager","sections":[],"depth":2},{"title":"Registry","local":"lighteval.tasks.registry.Registry","sections":[],"depth":2},{"title":"Doc","local":"lighteval.tasks.requests.Doc","sections":[],"depth":2},{"title":"Datasets","local":"lighteval.data.DynamicBatchDataset","sections":[],"depth":2}],"depth":1}';function qn(j){return bn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rn extends yn{constructor(f){super(),kn(this,f,qn,Dn,$n,{})}}export{Rn as component};