lighteval/pr_744/en/_app/immutable/nodes/13.aa2da103.js

import{s as Zo,c as So,u as Xo,g as Yo,d as Qo,o as Uo,b as Ko,n as er}from"../chunks/scheduler.7da89386.js";import{S as Ro,i as Ho,g as a,r as p,s as o,h as s,j as u,u as g,f as t,c as r,k as h,a as i,y as n,v as c,B as tr,d,t as m,w as f,A as nr,x}from"../chunks/index.20910acc.js";import{g as or,D as b}from"../chunks/Docstring.803c9cb0.js";import{C as rr}from"../chunks/CodeBlock.143bd81e.js";import{I as lr,H as A,E as ar}from"../chunks/index.c9cd5e8b.js";const{window:sr}=or;function ir(E){let v,w,D,M,T,$,N,P,k;M=new lr({props:{classNames:"text-smd"}});const j=E[4].default,_=So(j,E,E[3],null);return{c(){v=a("div"),w=a("a"),D=a("span"),p(M.$$.fragment),$=o(),_&&_.c(),this.h()},l(y){v=s(y,"DIV",{class:!0});var q=u(v);w=s(q,"A",{id:!0,class:!0,href:!0});var J=u(w);D=s(J,"SPAN",{});var W=u(D);g(M.$$.fragment,W),W.forEach(t),J.forEach(t),$=r(q),_&&_.l(q),q.forEach(t),this.h()},h(){h(w,"id",E[0]),h(w,"class","header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(w,"href",T=`#${E[0]}`),h(v,"class","relative group rounded-md")},m(y,q){i(y,v,q),n(v,w),n(w,D),c(M,D,null),n(v,$),_&&_.m(v,null),E[5](v),N=!0,P||(k=tr(sr,"hashchange",E[2]),P=!0)},p(y,[q]){(!N||q&1)&&h(w,"id",y[0]),(!N||q&1&&T!==(T=`#${y[0]}`))&&h(w,"href",T),_&&_.p&&(!N||q&8)&&Xo(_,j,y,y[3],N?Qo(j,y[3],q,null):Yo(y[3]),null)},i(y){N||(d(M.$$.fragment,y),d(_,y),N=!0)},o(y){m(M.$$.fragment,y),m(_,y),N=!1},d(y){y&&t(v),f(M),_&&_.d(y),E[5](null),P=!1,k()}}}const Oo="bg-yellow-50 dark:bg-[#494a3d]";function dr(E,v,w){let{$$slots:D={},$$scope:M}=v,{anchor:T}=v,$;function N(){const{hash:k}=window.location,j=k.substring(1);$&&$.classList.remove(...Oo.split(" ")),j===T&&$.classList.add(...Oo.split(" "))}Uo(()=>{N()});function P(k){Ko[k?"unshift":"push"](()=>{$=k,w(1,$)})}return E.$$set=k=>{"anchor"in k&&w(0,T=k.anchor),"$$scope"in k&&w(3,M=k.$$scope)},[T,$,N,M,D,P]}class mr extends Ro{constructor(v){super(),Ho(this,v,dr,ir,Zo,{anchor:0})}}function pr(E){let v,w="Example usage:",D,M,T;return M=new rr({props:{code:"JTIzJTIwRGVmaW5lJTIwY29uZmlnJTBBY29uZmlnJTIwJTNEJTIwQ3VzdG9tTW9kZWxDb25maWcoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJteS1jdXN0b20tbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjBtb2RlbF9kZWZpbml0aW9uX2ZpbGVfcGF0aCUzRCUyMnBhdGglMkZ0byUyRm15X21vZGVsLnB5JTIyJTBBKSUwQSUwQSUyMyUyMEV4YW1wbGUlMjBjdXN0b20lMjBtb2RlbCUyMGZpbGUlMjAobXlfbW9kZWwucHkpJTNBJTBBZnJvbSUyMGxpZ2h0ZXZhbC5tb2RlbHMuYWJzdHJhY3RfbW9kZWwlMjBpbXBvcnQlMjBMaWdodGV2YWxNb2RlbCUwQSUwQWNsYXNzJTIwTXlDdXN0b21Nb2RlbChMaWdodGV2YWxNb2RlbCklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmJTJDJTIwY29uZmlnJTJDJTIwZW52X2NvbmZpZyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdXBlcigpLl9faW5pdF9fKGNvbmZpZyUyQyUyMGVudl9jb25maWcpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwQ3VzdG9tJTIwaW5pdGlhbGl6YXRpb24uLi4lMEElMEElMjAlMjAlMjAlMjBkZWYlMjBncmVlZHlfdW50aWwoc2VsZiUyQyUyMCphcmdzJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwQ3VzdG9tJTIwZ2VuZXJhdGlvbiUyMGxvZ2ljLi4uJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGFzcw==",highlighted:`<span class="hljs-comment"># Define config</span>
config = CustomModelConfig(
    model=<span class="hljs-string">&quot;my-custom-model&quot;</span>,
    model_definition_file_path=<span class="hljs-string">&quot;path/to/my_model.py&quot;</span>
)

<span class="hljs-comment"># Example custom model file (my_model.py):</span>
<span class="hljs-keyword">from</span> lighteval.models.abstract_model <span class="hljs-keyword">import</span> LightevalModel

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyCustomModel</span>(<span class="hljs-title class_ inherited__">LightevalModel</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config, env_config</span>):
        <span class="hljs-built_in">super</span>().__init__(config, env_config)
        <span class="hljs-comment"># Custom initialization...</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">greedy_until</span>(<span class="hljs-params">self, *args, **kwargs</span>):
        <span class="hljs-comment"># Custom generation logic...</span>
        <span class="hljs-keyword">pass</span>`,wrap:!1}}),{c(){v=a("p"),v.textContent=w,D=o(),p(M.$$.fragment)},l($){v=s($,"P",{"data-svelte-h":!0}),x(v)!=="svelte-1ni337v"&&(v.textContent=w),D=r($),g(M.$$.fragment,$)},m($,N){i($,v,N),i($,D,N),c(M,$,N),T=!0},p:er,i($){T||(d(M.$$.fragment,$),T=!0)},o($){m(M.$$.fragment,$),T=!1},d($){$&&(t(v),t(D)),f(M,$)}}}function gr(E){let v,w,D,M,T,$,N,P,k,j,_,y,q,J,W,Mn,nt,mo="Clean up operations if needed, such as closing an endpoint.",Nn,U,ae,wn,ot,po="Generates responses using a greedy decoding strategy until certain ending conditions are met.",Tn,R,se,Cn,rt,go="Generates responses using a greedy decoding strategy until certain ending conditions are met.",kn,H,ie,In,lt,co=`Tokenize the context and continuation and compute the log likelihood of those
tokenized sequences.`,Ln,S,de,Dn,at,fo="This function is used to compute the log likelihood of the context for perplexity metrics.",En,X,me,Gn,st,ho=`Tokenize the context and continuation and compute the log likelihood of those
tokenized sequences.`,qn,B,pe,zn,it,uo="Encodes a context, continuation pair by taking care of the spaces in between.",An,dt,_o=`The advantage of pairwise is:
1) It better aligns with how LLM predicts tokens
2) Works in case len(tok(context,cont)) != len(tok(context)) + len(tok(continuation)).
E.g this can happen for chinese if no space is used between context/continuation`,Lt,ge,Dt,ce,Et,V,fe,Jn,mt,vo="Base configuration class for models.",Vn,pt,bo=`Methods:
<strong>post_init</strong>(): Performs post-initialization checks on the configuration.
_init_configs(model_name, env_config): Initializes the model configuration.
init_configs(env_config): Initializes the model configuration using the environment configuration.
get_model_sha(): Retrieves the SHA of the model.`,Gt,I,he,Pn,Y,ue,jn,gt,$o="Generates responses using a greedy decoding strategy until certain ending conditions are met.",Wn,Q,_e,Bn,ct,yo="Compute all the parameters related to model_parallel",Fn,K,ve,On,ft,xo=`Tokenize the context and continuation and compute the log likelihood of those
tokenized sequences.`,Zn,ee,be,Un,ht,Mo=`Tokenize the context and continuation and compute the log likelihood of those
tokenized sequences.`,Rn,te,$e,Hn,ut,No="Pads the <code>output_tensor</code> to the maximum length and gathers the lengths across processes.",Sn,ne,ye,Xn,_t,wo=`Tokenize a batch of inputs and return also the length, truncations and padding.
This step is done manually since we tokenize log probability inputs together with their continuation,
to manage possible extra spaces added at the start by tokenizers, see tok_encode_pair.`,qt,xe,zt,Me,Ne,At,we,Te,Jt,Ce,Vt,ke,Ie,Pt,Le,De,jt,Ee,Wt,Ge,Bt,qe,ze,Ft,Ae,Je,Ot,F,Ve,Yn,vt,To=`InferenceEndpointModels can be used both with the free inference client, or with inference
endpoints, which will use text-generation-inference to deploy your model for the duration of the evaluation.`,Zt,Pe,Ut,je,We,Rt,Be,Fe,Ht,Oe,St,C,Ze,Qn,bt,Co="Configuration class for loading custom model implementations in Lighteval.",Kn,$t,ko=`This config allows users to define and load their own model implementations by specifying
a Python file containing a custom model class that inherits from LightevalModel.`,eo,yt,Io=`The custom model file should contain exactly one class that inherits from LightevalModel.
This class will be automatically detected and instantiated when loading the model.`,to,oe,no,xt,Lo="An example of a custom model can be found in <code>examples/custom_models/google_translate_model.py</code>.",oo,Mt,Do="Notes:",ro,Nt,Eo="<li>The custom model class must inherit from LightevalModel and implement all required methods</li> <li>Only one class inheriting from LightevalModel should be defined in the file</li> <li>The model file is dynamically loaded at runtime, so ensure all dependencies are available</li> <li>Exercise caution when loading custom model files as they can execute arbitrary code</li>",Xt,Ue,Yt,O,Re,lo,re,He,ao,wt,Go="Generates responses using a greedy decoding strategy until certain ending conditions are met.",Qt,Se,Kt,Xe,en,Ye,Qe,tn,Z,Ke,so,le,et,io,Tt,qo="Generates responses using a greedy decoding strategy until certain ending conditions are met.",nn,tt,on,It,rn;return T=new A({props:{title:"Models",local:"models",headingTag:"h1"}}),N=new A({props:{title:"Model",local:"model",headingTag:"h2"}}),k=new A({props:{title:"LightevalModel",local:"lighteval.models.abstract_model.LightevalModel",headingTag:"h3"}}),y=new b({props:{name:"class lighteval.models.abstract_model.LightevalModel",anchor:"lighteval.models.abstract_model.LightevalModel",parameters:[],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/abstract_model.py#L57"}}),W=new b({props:{name:"cleanup",anchor:"lighteval.models.abstract_model.LightevalModel.cleanup",parameters:[],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/abstract_model.py#L62"}}),ae=new b({props:{name:"greedy_until",anchor:"lighteval.models.abstract_model.LightevalModel.greedy_until",parameters:[{name:"requests",val:": list"}],parametersDescription:[{anchor:"lighteval.models.abstract_model.LightevalModel.greedy_until.requests",description:"<strong>requests</strong> (list[Request]) &#x2014; list of requests containing the context and ending conditions.",name:"requests"},{anchor:"lighteval.models.abstract_model.LightevalModel.greedy_until.disable_tqdm",description:"<strong>disable_tqdm</strong> (bool, optional) &#x2014; Whether to disable the progress bar. Defaults to False.",name:"disable_tqdm"},{anchor:"lighteval.models.abstract_model.LightevalModel.greedy_until.override_bs",description:"<strong>override_bs</strong> (int, optional) &#x2014; Override the batch size for generation. Defaults to None.",name:"override_bs"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/abstract_model.py#L105",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list of generated responses.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[GenerativeResponse]</p>
`}}),se=new b({props:{name:"greedy_until_multi_turn",anchor:"lighteval.models.abstract_model.LightevalModel.greedy_until_multi_turn",parameters:[{name:"requests",val:": list"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/abstract_model.py#L99"}}),ie=new b({props:{name:"loglikelihood",anchor:"lighteval.models.abstract_model.LightevalModel.loglikelihood",parameters:[{name:"requests",val:": list"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/abstract_model.py#L123"}}),de=new b({props:{name:"loglikelihood_rolling",anchor:"lighteval.models.abstract_model.LightevalModel.loglikelihood_rolling",parameters:[{name:"requests",val:": list"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/abstract_model.py#L130"}}),me=new b({props:{name:"loglikelihood_single_token",anchor:"lighteval.models.abstract_model.LightevalModel.loglikelihood_single_token",parameters:[{name:"requests",val:": list"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/abstract_model.py#L135"}}),pe=new b({props:{name:"tok_encode_pair",anchor:"lighteval.models.abstract_model.LightevalModel.tok_encode_pair",parameters:[{name:"context",val:""},{name:"continuation",val:""},{name:"pairwise",val:": bool = False"}],parametersDescription:[{anchor:"lighteval.models.abstract_model.LightevalModel.tok_encode_pair.context",description:"<strong>context</strong> (str) &#x2014; The context string to be encoded.",name:"context"},{anchor:"lighteval.models.abstract_model.LightevalModel.tok_encode_pair.continuation",description:"<strong>continuation</strong> (str) &#x2014; The continuation string to be encoded.",name:"continuation"},{anchor:"lighteval.models.abstract_model.LightevalModel.tok_encode_pair.pairwise",description:`<strong>pairwise</strong> (bool) &#x2014;
If True, encode context and continuation separately.
If False, encode them together and then split.`,name:"pairwise"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/abstract_model.py#L157",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A tuple containing the encoded context and continuation.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Tuple[TokenSequence, TokenSequence]</p>
`}}),ge=new A({props:{title:"Accelerate and Transformers Models",local:"accelerate-and-transformers-models",headingTag:"h2"}}),ce=new A({props:{title:"TransformersModel",local:"lighteval.models.transformers.transformers_model.TransformersModelConfig",headingTag:"h3"}}),fe=new b({props:{name:"class lighteval.models.transformers.transformers_model.TransformersModelConfig",anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig",parameters:[{name:"generation_parameters",val:": GenerationParameters = GenerationParameters(early_stopping=None, repetition_penalty=None, frequency_penalty=None, length_penalty=None, presence_penalty=None, max_new_tokens=None, min_new_tokens=None, seed=None, stop_tokens=None, temperature=None, top_k=None, min_p=None, top_p=None, truncate_prompt=None, response_format=None)"},{name:"model_name",val:": str"},{name:"tokenizer",val:": str | None = None"},{name:"subfolder",val:": str | None = None"},{name:"revision",val:": str = 'main'"},{name:"batch_size",val:": typing.Optional[typing.Annotated[int, Gt(gt=0)]] = None"},{name:"generation_size",val:": typing.Annotated[int, Gt(gt=0)] = 256"},{name:"max_length",val:": typing.Optional[typing.Annotated[int, Gt(gt=0)]] = None"},{name:"add_special_tokens",val:": bool = True"},{name:"model_parallel",val:": bool | None = None"},{name:"dtype",val:": str | None = None"},{name:"device",val:": typing.Union[int, str] = 'cuda'"},{name:"trust_remote_code",val:": bool = False"},{name:"use_chat_template",val:": bool = False"},{name:"compile",val:": bool = False"},{name:"multichoice_continuations_start_space",val:": bool | None = None"},{name:"pairwise_tokenization",val:": bool = False"}],parametersDescription:[{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.model_name",description:`<strong>model_name</strong> (str) &#x2014;
HuggingFace Hub model ID name or the path to a pre-trained
model to load. This is effectively the <code>pretrained_model_name_or_path</code>
argument of <code>from_pretrained</code> in the HuggingFace <code>transformers</code> API.`,name:"model_name"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.accelerator",description:"<strong>accelerator</strong> (Accelerator) &#x2014; accelerator to use for model training.",name:"accelerator"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.tokenizer",description:`<strong>tokenizer</strong> (Optional[str]) &#x2014; HuggingFace Hub tokenizer ID that will be
used for tokenization.`,name:"tokenizer"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.multichoice_continuations_start_space",description:`<strong>multichoice_continuations_start_space</strong> (Optional[bool]) &#x2014; Whether to add a
space at the start of each continuation in multichoice generation.
For example, context: &#x201C;What is the capital of France?&#x201D; and choices: &#x201C;Paris&#x201D;, &#x201C;London&#x201D;.
Will be tokenized as: &#x201C;What is the capital of France? Paris&#x201D; and &#x201C;What is the capital of France? London&#x201D;.
True adds a space, False strips a space, None does nothing`,name:"multichoice_continuations_start_space"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.pairwise_tokenization",description:"<strong>pairwise_tokenization</strong> (bool) &#x2014; Whether to tokenize the context and continuation as separately or together.",name:"pairwise_tokenization"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.subfolder",description:"<strong>subfolder</strong> (Optional[str]) &#x2014; The subfolder within the model repository.",name:"subfolder"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.revision",description:"<strong>revision</strong> (str) &#x2014; The revision of the model.",name:"revision"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.batch_size",description:"<strong>batch_size</strong> (int) &#x2014; The batch size for model training.",name:"batch_size"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.max_gen_toks",description:"<strong>max_gen_toks</strong> (Optional[int]) &#x2014; The maximum number of tokens to generate.",name:"max_gen_toks"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.max_length",description:"<strong>max_length</strong> (Optional[int]) &#x2014; The maximum length of the generated output.",name:"max_length"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.add_special_tokens",description:`<strong>add_special_tokens</strong> (bool, optional, defaults to True) &#x2014; Whether to add special tokens to the input sequences.
If <code>None</code>, the default value will be set to <code>True</code> for seq2seq models (e.g. T5) and
<code>False</code> for causal models.`,name:"add_special_tokens"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.model_parallel",description:`<strong>model_parallel</strong> (bool, optional, defaults to None) &#x2014;
True/False: force to use or not the <code>accelerate</code> library to load a large
model across multiple devices.
Default: None which corresponds to comparing the number of processes with
the number of GPUs. If it&#x2019;s smaller =&gt; model-parallelism, else not.`,name:"model_parallel"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.dtype",description:`<strong>dtype</strong> (Union[str, torch.dtype], optional, defaults to None) &#x2014;):
Converts the model weights to <code>dtype</code>, if specified. Strings get
converted to <code>torch.dtype</code> objects (e.g. <code>float16</code> -&gt; <code>torch.float16</code>).
Use <code>dtype=&quot;auto&quot;</code> to derive the type from the model&#x2019;s weights.`,name:"dtype"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.device",description:"<strong>device</strong> (Union[int, str]) &#x2014; device to use for model training.",name:"device"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.quantization_config",description:`<strong>quantization_config</strong> (Optional[BitsAndBytesConfig]) &#x2014; quantization
configuration for the model, manually provided to load a normally floating point
model at a quantized precision. Needed for 4-bit and 8-bit precision.`,name:"quantization_config"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.trust_remote_code",description:`<strong>trust_remote_code</strong> (bool) &#x2014; Whether to trust remote code during model
loading.`,name:"trust_remote_code"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.generation_parameters",description:"<strong>generation_parameters</strong> (GenerationParameters) &#x2014; Range of parameters which will affect the generation.",name:"generation_parameters"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModelConfig.generation_config",description:"<strong>generation_config</strong> (GenerationConfig) &#x2014; GenerationConfig object (only passed during manual creation)",name:"generation_config"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/transformers_model.py#L83"}}),he=new b({props:{name:"class lighteval.models.transformers.transformers_model.TransformersModel",anchor:"lighteval.models.transformers.transformers_model.TransformersModel",parameters:[{name:"config",val:": TransformersModelConfig"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/transformers_model.py#L180"}}),ue=new b({props:{name:"greedy_until",anchor:"lighteval.models.transformers.transformers_model.TransformersModel.greedy_until",parameters:[{name:"requests",val:": list"}],parametersDescription:[{anchor:"lighteval.models.transformers.transformers_model.TransformersModel.greedy_until.requests",description:"<strong>requests</strong> (list[Request]) &#x2014; list of requests containing the context and ending conditions.",name:"requests"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModel.greedy_until.override_bs",description:"<strong>override_bs</strong> (int, optional) &#x2014; Override the batch size for generation. Defaults to None.",name:"override_bs"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/transformers_model.py#L511",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list of generated responses.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[GenerativeResponse]</p>
`}}),_e=new b({props:{name:"init_model_parallel",anchor:"lighteval.models.transformers.transformers_model.TransformersModel.init_model_parallel",parameters:[{name:"model_parallel",val:": bool | None = None"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/transformers_model.py#L321"}}),ve=new b({props:{name:"loglikelihood",anchor:"lighteval.models.transformers.transformers_model.TransformersModel.loglikelihood",parameters:[{name:"requests",val:": list"}],parametersDescription:[{anchor:"lighteval.models.transformers.transformers_model.TransformersModel.loglikelihood.requests",description:"<strong>requests</strong> (list[Tuple[str, dict]]) &#x2014; <em>description</em>",name:"requests"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/transformers_model.py#L714",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><em>description</em></p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[Tuple[float, bool]]</p>
`}}),be=new b({props:{name:"loglikelihood_single_token",anchor:"lighteval.models.transformers.transformers_model.TransformersModel.loglikelihood_single_token",parameters:[{name:"requests",val:": list"}],parametersDescription:[{anchor:"lighteval.models.transformers.transformers_model.TransformersModel.loglikelihood_single_token.requests",description:"<strong>requests</strong> (list[Tuple[str, dict]]) &#x2014; <em>description</em>",name:"requests"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/transformers_model.py#L969",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><em>description</em></p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[Tuple[float, bool]]</p>
`}}),$e=new b({props:{name:"pad_and_gather",anchor:"lighteval.models.transformers.transformers_model.TransformersModel.pad_and_gather",parameters:[{name:"output_tensor",val:": Tensor"},{name:"drop_last_samples",val:": bool = True"},{name:"num_samples",val:": int = None"}],parametersDescription:[{anchor:"lighteval.models.transformers.transformers_model.TransformersModel.pad_and_gather.output_tensor",description:"<strong>output_tensor</strong> (torch.Tensor) &#x2014; The output tensor to be padded.",name:"output_tensor"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModel.pad_and_gather.drop_last_samples",description:"<strong>drop_last_samples</strong> (bool, optional) &#x2014; Whether to drop the last samples during gathering.",name:"drop_last_samples"},{anchor:"lighteval.models.transformers.transformers_model.TransformersModel.pad_and_gather.Last",description:`<strong>Last</strong> samples are dropped when the number of samples is not divisible by the number of processes. &#x2014;
Defaults to True.`,name:"Last"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/transformers_model.py#L933",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The padded output tensor and the gathered length tensor.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>torch.Tensor</p>
`}}),ye=new b({props:{name:"prepare_batch_logprob",anchor:"lighteval.models.transformers.transformers_model.TransformersModel.prepare_batch_logprob",parameters:[{name:"batch",val:": list"},{name:"padding_length",val:": int"},{name:"max_context",val:": typing.Optional[int] = None"},{name:"single_token",val:": bool = False"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/transformers_model.py#L872"}}),xe=new A({props:{title:"AdapterModel",local:"lighteval.models.transformers.adapter_model.AdapterModelConfig",headingTag:"h3"}}),Ne=new b({props:{name:"class lighteval.models.transformers.adapter_model.AdapterModelConfig",anchor:"lighteval.models.transformers.adapter_model.AdapterModelConfig",parameters:[{name:"generation_parameters",val:": GenerationParameters = GenerationParameters(early_stopping=None, repetition_penalty=None, frequency_penalty=None, length_penalty=None, presence_penalty=None, max_new_tokens=None, min_new_tokens=None, seed=None, stop_tokens=None, temperature=None, top_k=None, min_p=None, top_p=None, truncate_prompt=None, response_format=None)"},{name:"model_name",val:": str"},{name:"tokenizer",val:": str | None = None"},{name:"subfolder",val:": str | None = None"},{name:"revision",val:": str = 'main'"},{name:"batch_size",val:": typing.Optional[typing.Annotated[int, Gt(gt=0)]] = None"},{name:"generation_size",val:": typing.Annotated[int, Gt(gt=0)] = 256"},{name:"max_length",val:": typing.Optional[typing.Annotated[int, Gt(gt=0)]] = None"},{name:"add_special_tokens",val:": bool = True"},{name:"model_parallel",val:": bool | None = None"},{name:"dtype",val:": str | None = None"},{name:"device",val:": typing.Union[int, str] = 'cuda'"},{name:"trust_remote_code",val:": bool = False"},{name:"use_chat_template",val:": bool = False"},{name:"compile",val:": bool = False"},{name:"multichoice_continuations_start_space",val:": bool | None = None"},{name:"pairwise_tokenization",val:": bool = False"},{name:"base_model",val:": str"},{name:"adapter_weights",val:": bool"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/adapter_model.py#L41"}}),Te=new b({props:{name:"class lighteval.models.transformers.adapter_model.AdapterModel",anchor:"lighteval.models.transformers.adapter_model.AdapterModel",parameters:[{name:"config",val:": TransformersModelConfig"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/adapter_model.py#L51"}}),Ce=new A({props:{title:"DeltaModel",local:"lighteval.models.transformers.delta_model.DeltaModelConfig",headingTag:"h3"}}),Ie=new b({props:{name:"class lighteval.models.transformers.delta_model.DeltaModelConfig",anchor:"lighteval.models.transformers.delta_model.DeltaModelConfig",parameters:[{name:"generation_parameters",val:": GenerationParameters = GenerationParameters(early_stopping=None, repetition_penalty=None, frequency_penalty=None, length_penalty=None, presence_penalty=None, max_new_tokens=None, min_new_tokens=None, seed=None, stop_tokens=None, temperature=None, top_k=None, min_p=None, top_p=None, truncate_prompt=None, response_format=None)"},{name:"model_name",val:": str"},{name:"tokenizer",val:": str | None = None"},{name:"subfolder",val:": str | None = None"},{name:"revision",val:": str = 'main'"},{name:"batch_size",val:": typing.Optional[typing.Annotated[int, Gt(gt=0)]] = None"},{name:"generation_size",val:": typing.Annotated[int, Gt(gt=0)] = 256"},{name:"max_length",val:": typing.Optional[typing.Annotated[int, Gt(gt=0)]] = None"},{name:"add_special_tokens",val:": bool = True"},{name:"model_parallel",val:": bool | None = None"},{name:"dtype",val:": str | None = None"},{name:"device",val:": typing.Union[int, str] = 'cuda'"},{name:"trust_remote_code",val:": bool = False"},{name:"use_chat_template",val:": bool = False"},{name:"compile",val:": bool = False"},{name:"multichoice_continuations_start_space",val:": bool | None = None"},{name:"pairwise_tokenization",val:": bool = False"},{name:"base_model",val:": str"},{name:"delta_weights",val:": bool"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/delta_model.py#L37"}}),De=new b({props:{name:"class lighteval.models.transformers.delta_model.DeltaModel",anchor:"lighteval.models.transformers.delta_model.DeltaModel",parameters:[{name:"config",val:": TransformersModelConfig"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/transformers/delta_model.py#L46"}}),Ee=new A({props:{title:"Endpoints-based Models",local:"endpoints-based-models",headingTag:"h2"}}),Ge=new A({props:{title:"InferenceEndpointModel",local:"lighteval.models.endpoints.endpoint_model.InferenceEndpointModelConfig",headingTag:"h3"}}),ze=new b({props:{name:"class lighteval.models.endpoints.endpoint_model.InferenceEndpointModelConfig",anchor:"lighteval.models.endpoints.endpoint_model.InferenceEndpointModelConfig",parameters:[{name:"generation_parameters",val:": GenerationParameters = GenerationParameters(early_stopping=None, repetition_penalty=None, frequency_penalty=None, length_penalty=None, presence_penalty=None, max_new_tokens=None, min_new_tokens=None, seed=None, stop_tokens=None, temperature=None, top_k=None, min_p=None, top_p=None, truncate_prompt=None, response_format=None)"},{name:"endpoint_name",val:": str | None = None"},{name:"model_name",val:": str | None = None"},{name:"reuse_existing",val:": bool = False"},{name:"accelerator",val:": str = 'gpu'"},{name:"dtype",val:": str | None = None"},{name:"vendor",val:": str = 'aws'"},{name:"region",val:": str = 'us-east-1'"},{name:"instance_size",val:": str | None = None"},{name:"instance_type",val:": str | None = None"},{name:"framework",val:": str = 'pytorch'"},{name:"endpoint_type",val:": str = 'protected'"},{name:"add_special_tokens",val:": bool = True"},{name:"revision",val:": str = 'main'"},{name:"namespace",val:": str | None = None"},{name:"image_url",val:": str | None = None"},{name:"env_vars",val:": dict | None = None"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/endpoints/endpoint_model.py#L82"}}),Je=new b({props:{name:"class lighteval.models.endpoints.endpoint_model.ServerlessEndpointModelConfig",anchor:"lighteval.models.endpoints.endpoint_model.ServerlessEndpointModelConfig",parameters:[{name:"generation_parameters",val:": GenerationParameters = GenerationParameters(early_stopping=None, repetition_penalty=None, frequency_penalty=None, length_penalty=None, presence_penalty=None, max_new_tokens=None, min_new_tokens=None, seed=None, stop_tokens=None, temperature=None, top_k=None, min_p=None, top_p=None, truncate_prompt=None, response_format=None)"},{name:"model_name",val:": str"},{name:"add_special_tokens",val:": bool = True"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/endpoints/endpoint_model.py#L77"}}),Ve=new b({props:{name:"class lighteval.models.endpoints.endpoint_model.InferenceEndpointModel",anchor:"lighteval.models.endpoints.endpoint_model.InferenceEndpointModel",parameters:[{name:"config",val:": typing.Union[lighteval.models.endpoints.endpoint_model.InferenceEndpointModelConfig, lighteval.models.endpoints.endpoint_model.ServerlessEndpointModelConfig]"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/endpoints/endpoint_model.py#L130"}}),Pe=new A({props:{title:"TGI ModelClient",local:"lighteval.models.endpoints.tgi_model.TGIModelConfig",headingTag:"h3"}}),We=new b({props:{name:"class lighteval.models.endpoints.tgi_model.TGIModelConfig",anchor:"lighteval.models.endpoints.tgi_model.TGIModelConfig",parameters:[{name:"generation_parameters",val:": GenerationParameters = GenerationParameters(early_stopping=None, repetition_penalty=None, frequency_penalty=None, length_penalty=None, presence_penalty=None, max_new_tokens=None, min_new_tokens=None, seed=None, stop_tokens=None, temperature=None, top_k=None, min_p=None, top_p=None, truncate_prompt=None, response_format=None)"},{name:"inference_server_address",val:": str | None"},{name:"inference_server_auth",val:": str | None"},{name:"model_name",val:": str | None"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/endpoints/tgi_model.py#L49"}}),Fe=new b({props:{name:"class lighteval.models.endpoints.tgi_model.ModelClient",anchor:"lighteval.models.endpoints.tgi_model.ModelClient",parameters:[{name:"config",val:": TGIModelConfig"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/endpoints/tgi_model.py#L57"}}),Oe=new A({props:{title:"Custom Model",local:"lighteval.models.custom.custom_model.CustomModelConfig",headingTag:"h3"}}),Ze=new b({props:{name:"class lighteval.models.custom.custom_model.CustomModelConfig",anchor:"lighteval.models.custom.custom_model.CustomModelConfig",parameters:[{name:"generation_parameters",val:": GenerationParameters = GenerationParameters(early_stopping=None, repetition_penalty=None, frequency_penalty=None, length_penalty=None, presence_penalty=None, max_new_tokens=None, min_new_tokens=None, seed=None, stop_tokens=None, temperature=None, top_k=None, min_p=None, top_p=None, truncate_prompt=None, response_format=None)"},{name:"model_name",val:": str"},{name:"model_definition_file_path",val:": str"}],parametersDescription:[{anchor:"lighteval.models.custom.custom_model.CustomModelConfig.model",description:`<strong>model</strong> (str) &#x2014;
An identifier for the model. This can be used to track which model was evaluated
in the results and logs.`,name:"model"},{anchor:"lighteval.models.custom.custom_model.CustomModelConfig.model_definition_file_path",description:`<strong>model_definition_file_path</strong> (str) &#x2014;
Path to a Python file containing the custom model implementation. This file must
define exactly one class that inherits from LightevalModel. The class should
implement all required methods from the LightevalModel interface.`,name:"model_definition_file_path"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/custom/custom_model.py#L26"}}),oe=new mr({props:{anchor:"lighteval.models.custom.custom_model.CustomModelConfig.example",$$slots:{default:[pr]},$$scope:{ctx:E}}}),Ue=new A({props:{title:"Open AI Models",local:"lighteval.models.endpoints.openai_model.OpenAIClient",headingTag:"h3"}}),Re=new b({props:{name:"class lighteval.models.endpoints.openai_model.OpenAIClient",anchor:"lighteval.models.endpoints.openai_model.OpenAIClient",parameters:[{name:"config",val:": OpenAIModelConfig"},{name:"env_config",val:""}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/endpoints/openai_model.py#L87"}}),He=new b({props:{name:"greedy_until",anchor:"lighteval.models.endpoints.openai_model.OpenAIClient.greedy_until",parameters:[{name:"requests",val:": list"},{name:"override_bs",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"lighteval.models.endpoints.openai_model.OpenAIClient.greedy_until.requests",description:"<strong>requests</strong> (list[Request]) &#x2014; list of requests containing the context and ending conditions.",name:"requests"},{anchor:"lighteval.models.endpoints.openai_model.OpenAIClient.greedy_until.override_bs",description:"<strong>override_bs</strong> (int, optional) &#x2014; Override the batch size for generation. Defaults to None.",name:"override_bs"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/endpoints/openai_model.py#L166",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list of generated responses.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[GenerativeResponse]</p>
`}}),Se=new A({props:{title:"VLLM Model",local:"vllm-model",headingTag:"h2"}}),Xe=new A({props:{title:"VLLMModel",local:"lighteval.models.vllm.vllm_model.VLLMModelConfig",headingTag:"h3"}}),Qe=new b({props:{name:"class lighteval.models.vllm.vllm_model.VLLMModelConfig",anchor:"lighteval.models.vllm.vllm_model.VLLMModelConfig",parameters:[{name:"generation_parameters",val:": GenerationParameters = GenerationParameters(early_stopping=None, repetition_penalty=None, frequency_penalty=None, length_penalty=None, presence_penalty=None, max_new_tokens=None, min_new_tokens=None, seed=None, stop_tokens=None, temperature=None, top_k=None, min_p=None, top_p=None, truncate_prompt=None, response_format=None)"},{name:"model_name",val:": str"},{name:"revision",val:": str = 'main'"},{name:"dtype",val:": str = 'bfloat16'"},{name:"tensor_parallel_size",val:": typing.Annotated[int, Gt(gt=0)] = 1"},{name:"data_parallel_size",val:": typing.Annotated[int, Gt(gt=0)] = 1"},{name:"pipeline_parallel_size",val:": typing.Annotated[int, Gt(gt=0)] = 1"},{name:"gpu_memory_utilization",val:": typing.Annotated[float, Ge(ge=0)] = 0.9"},{name:"max_model_length",val:": typing.Optional[typing.Annotated[int, Gt(gt=0)]] = None"},{name:"quantization",val:": str | None = None"},{name:"load_format",val:": str | None = None"},{name:"swap_space",val:": typing.Annotated[int, Gt(gt=0)] = 4"},{name:"seed",val:": typing.Annotated[int, Ge(ge=0)] = 1234"},{name:"trust_remote_code",val:": bool = False"},{name:"use_chat_template",val:": bool = False"},{name:"add_special_tokens",val:": bool = True"},{name:"multichoice_continuations_start_space",val:": bool = True"},{name:"pairwise_tokenization",val:": bool = False"},{name:"max_num_seqs",val:": typing.Annotated[int, Gt(gt=0)] = 128"},{name:"max_num_batched_tokens",val:": typing.Annotated[int, Gt(gt=0)] = 2048"},{name:"subfolder",val:": str | None = None"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/vllm/vllm_model.py#L75"}}),Ke=new b({props:{name:"class lighteval.models.vllm.vllm_model.VLLMModel",anchor:"lighteval.models.vllm.vllm_model.VLLMModel",parameters:[{name:"config",val:": VLLMModelConfig"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/vllm/vllm_model.py#L100"}}),et=new b({props:{name:"greedy_until",anchor:"lighteval.models.vllm.vllm_model.VLLMModel.greedy_until",parameters:[{name:"requests",val:": list"},{name:"override_bs",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"lighteval.models.vllm.vllm_model.VLLMModel.greedy_until.requests",description:"<strong>requests</strong> (list[Request]) &#x2014; list of requests containing the context and ending conditions.",name:"requests"},{anchor:"lighteval.models.vllm.vllm_model.VLLMModel.greedy_until.override_bs",description:"<strong>override_bs</strong> (int, optional) &#x2014; Override the batch size for generation. Defaults to None.",name:"override_bs"}],source:"https://github.com/huggingface/lighteval/blob/vr_744/src/lighteval/models/vllm/vllm_model.py#L212",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list of generated responses.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list[GenerateReturn]</p>
`}}),tt=new ar({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/package_reference/models.mdx"}}),{c(){v=a("meta"),w=o(),D=a("p"),M=o(),p(T.$$.fragment),$=o(),p(N.$$.fragment),P=o(),p(k.$$.fragment),j=o(),_=a("div"),p(y.$$.fragment),q=o(),J=a("div"),p(W.$$.fragment),Mn=o(),nt=a("p"),nt.textContent=mo,Nn=o(),U=a("div"),p(ae.$$.fragment),wn=o(),ot=a("p"),ot.textContent=po,Tn=o(),R=a("div"),p(se.$$.fragment),Cn=o(),rt=a("p"),rt.textContent=go,kn=o(),H=a("div"),p(ie.$$.fragment),In=o(),lt=a("p"),lt.textContent=co,Ln=o(),S=a("div"),p(de.$$.fragment),Dn=o(),at=a("p"),at.textContent=fo,En=o(),X=a("div"),p(me.$$.fragment),Gn=o(),st=a("p"),st.textContent=ho,qn=o(),B=a("div"),p(pe.$$.fragment),zn=o(),it=a("p"),it.textContent=uo,An=o(),dt=a("p"),dt.textContent=_o,Lt=o(),p(ge.$$.fragment),Dt=o(),p(ce.$$.fragment),Et=o(),V=a("div"),p(fe.$$.fragment),Jn=o(),mt=a("p"),mt.textContent=vo,Vn=o(),pt=a("p"),pt.innerHTML=bo,Gt=o(),I=a("div"),p(he.$$.fragment),Pn=o(),Y=a("div"),p(ue.$$.fragment),jn=o(),gt=a("p"),gt.textContent=$o,Wn=o(),Q=a("div"),p(_e.$$.fragment),Bn=o(),ct=a("p"),ct.textContent=yo,Fn=o(),K=a("div"),p(ve.$$.fragment),On=o(),ft=a("p"),ft.textContent=xo,Zn=o(),ee=a("div"),p(be.$$.fragment),Un=o(),ht=a("p"),ht.textContent=Mo,Rn=o(),te=a("div"),p($e.$$.fragment),Hn=o(),ut=a("p"),ut.innerHTML=No,Sn=o(),ne=a("div"),p(ye.$$.fragment),Xn=o(),_t=a("p"),_t.textContent=wo,qt=o(),p(xe.$$.fragment),zt=o(),Me=a("div"),p(Ne.$$.fragment),At=o(),we=a("div"),p(Te.$$.fragment),Jt=o(),p(Ce.$$.fragment),Vt=o(),ke=a("div"),p(Ie.$$.fragment),Pt=o(),Le=a("div"),p(De.$$.fragment),jt=o(),p(Ee.$$.fragment),Wt=o(),p(Ge.$$.fragment),Bt=o(),qe=a("div"),p(ze.$$.fragment),Ft=o(),Ae=a("div"),p(Je.$$.fragment),Ot=o(),F=a("div"),p(Ve.$$.fragment),Yn=o(),vt=a("p"),vt.textContent=To,Zt=o(),p(Pe.$$.fragment),Ut=o(),je=a("div"),p(We.$$.fragment),Rt=o(),Be=a("div"),p(Fe.$$.fragment),Ht=o(),p(Oe.$$.fragment),St=o(),C=a("div"),p(Ze.$$.fragment),Qn=o(),bt=a("p"),bt.textContent=Co,Kn=o(),$t=a("p"),$t.textContent=ko,eo=o(),yt=a("p"),yt.textContent=Io,to=o(),p(oe.$$.fragment),no=o(),xt=a("p"),xt.innerHTML=Lo,oo=o(),Mt=a("p"),Mt.textContent=Do,ro=o(),Nt=a("ul"),Nt.innerHTML=Eo,Xt=o(),p(Ue.$$.fragment),Yt=o(),O=a("div"),p(Re.$$.fragment),lo=o(),re=a("div"),p(He.$$.fragment),ao=o(),wt=a("p"),wt.textContent=Go,Qt=o(),p(Se.$$.fragment),Kt=o(),p(Xe.$$.fragment),en=o(),Ye=a("div"),p(Qe.$$.fragment),tn=o(),Z=a("div"),p(Ke.$$.fragment),so=o(),le=a("div"),p(et.$$.fragment),io=o(),Tt=a("p"),Tt.textContent=qo,nn=o(),p(tt.$$.fragment),on=o(),It=a("p"),this.h()},l(e){const l=nr("svelte-u9bgzb",document.head);v=s(l,"META",{name:!0,content:!0}),l.forEach(t),w=r(e),D=s(e,"P",{}),u(D).forEach(t),M=r(e),g(T.$$.fragment,e),$=r(e),g(N.$$.fragment,e),P=r(e),g(k.$$.fragment,e),j=r(e),_=s(e,"DIV",{class:!0});var L=u(_);g(y.$$.fragment,L),q=r(L),J=s(L,"DIV",{class:!0});var ln=u(J);g(W.$$.fragment,ln),Mn=r(ln),nt=s(ln,"P",{"data-svelte-h":!0}),x(nt)!=="svelte-l9my6g"&&(nt.textContent=mo),ln.forEach(t),Nn=r(L),U=s(L,"DIV",{class:!0});var an=u(U);g(ae.$$.fragment,an),wn=r(an),ot=s(an,"P",{"data-svelte-h":!0}),x(ot)!=="svelte-1smaw30"&&(ot.textContent=po),an.forEach(t),Tn=r(L),R=s(L,"DIV",{class:!0});var sn=u(R);g(se.$$.fragment,sn),Cn=r(sn),rt=s(sn,"P",{"data-svelte-h":!0}),x(rt)!=="svelte-1smaw30"&&(rt.textContent=go),sn.forEach(t),kn=r(L),H=s(L,"DIV",{class:!0});var dn=u(H);g(ie.$$.fragment,dn),In=r(dn),lt=s(dn,"P",{"data-svelte-h":!0}),x(lt)!=="svelte-14ip3fh"&&(lt.textContent=co),dn.forEach(t),Ln=r(L),S=s(L,"DIV",{class:!0});var mn=u(S);g(de.$$.fragment,mn),Dn=r(mn),at=s(mn,"P",{"data-svelte-h":!0}),x(at)!=="svelte-1g4mbb"&&(at.textContent=fo),mn.forEach(t),En=r(L),X=s(L,"DIV",{class:!0});var pn=u(X);g(me.$$.fragment,pn),Gn=r(pn),st=s(pn,"P",{"data-svelte-h":!0}),x(st)!=="svelte-14ip3fh"&&(st.textContent=ho),pn.forEach(t),qn=r(L),B=s(L,"DIV",{class:!0});var Ct=u(B);g(pe.$$.fragment,Ct),zn=r(Ct),it=s(Ct,"P",{"data-svelte-h":!0}),x(it)!=="svelte-1sddtie"&&(it.textContent=uo),An=r(Ct),dt=s(Ct,"P",{"data-svelte-h":!0}),x(dt)!=="svelte-gax3uy"&&(dt.textContent=_o),Ct.forEach(t),L.forEach(t),Lt=r(e),g(ge.$$.fragment,e),Dt=r(e),g(ce.$$.fragment,e),Et=r(e),V=s(e,"DIV",{class:!0});var kt=u(V);g(fe.$$.fragment,kt),Jn=r(kt),mt=s(kt,"P",{"data-svelte-h":!0}),x(mt)!=="svelte-rh03fc"&&(mt.textContent=vo),Vn=r(kt),pt=s(kt,"P",{"data-svelte-h":!0}),x(pt)!=="svelte-mdcuay"&&(pt.innerHTML=bo),kt.forEach(t),Gt=r(e),I=s(e,"DIV",{class:!0});var z=u(I);g(he.$$.fragment,z),Pn=r(z),Y=s(z,"DIV",{class:!0});var gn=u(Y);g(ue.$$.fragment,gn),jn=r(gn),gt=s(gn,"P",{"data-svelte-h":!0}),x(gt)!=="svelte-1smaw30"&&(gt.textContent=$o),gn.forEach(t),Wn=r(z),Q=s(z,"DIV",{class:!0});var cn=u(Q);g(_e.$$.fragment,cn),Bn=r(cn),ct=s(cn,"P",{"data-svelte-h":!0}),x(ct)!=="svelte-1syquxe"&&(ct.textContent=yo),cn.forEach(t),Fn=r(z),K=s(z,"DIV",{class:!0});var fn=u(K);g(ve.$$.fragment,fn),On=r(fn),ft=s(fn,"P",{"data-svelte-h":!0}),x(ft)!=="svelte-14ip3fh"&&(ft.textContent=xo),fn.forEach(t),Zn=r(z),ee=s(z,"DIV",{class:!0});var hn=u(ee);g(be.$$.fragment,hn),Un=r(hn),ht=s(hn,"P",{"data-svelte-h":!0}),x(ht)!=="svelte-14ip3fh"&&(ht.textContent=Mo),hn.forEach(t),Rn=r(z),te=s(z,"DIV",{class:!0});var un=u(te);g($e.$$.fragment,un),Hn=r(un),ut=s(un,"P",{"data-svelte-h":!0}),x(ut)!=="svelte-no3bag"&&(ut.innerHTML=No),un.forEach(t),Sn=r(z),ne=s(z,"DIV",{class:!0});var _n=u(ne);g(ye.$$.fragment,_n),Xn=r(_n),_t=s(_n,"P",{"data-svelte-h":!0}),x(_t)!=="svelte-bu0ewe"&&(_t.textContent=wo),_n.forEach(t),z.forEach(t),qt=r(e),g(xe.$$.fragment,e),zt=r(e),Me=s(e,"DIV",{class:!0});var zo=u(Me);g(Ne.$$.fragment,zo),zo.forEach(t),At=r(e),we=s(e,"DIV",{class:!0});var Ao=u(we);g(Te.$$.fragment,Ao),Ao.forEach(t),Jt=r(e),g(Ce.$$.fragment,e),Vt=r(e),ke=s(e,"DIV",{class:!0});var Jo=u(ke);g(Ie.$$.fragment,Jo),Jo.forEach(t),Pt=r(e),Le=s(e,"DIV",{class:!0});var Vo=u(Le);g(De.$$.fragment,Vo),Vo.forEach(t),jt=r(e),g(Ee.$$.fragment,e),Wt=r(e),g(Ge.$$.fragment,e),Bt=r(e),qe=s(e,"DIV",{class:!0});var Po=u(qe);g(ze.$$.fragment,Po),Po.forEach(t),Ft=r(e),Ae=s(e,"DIV",{class:!0});var jo=u(Ae);g(Je.$$.fragment,jo),jo.forEach(t),Ot=r(e),F=s(e,"DIV",{class:!0});var vn=u(F);g(Ve.$$.fragment,vn),Yn=r(vn),vt=s(vn,"P",{"data-svelte-h":!0}),x(vt)!=="svelte-1nzgk8o"&&(vt.textContent=To),vn.forEach(t),Zt=r(e),g(Pe.$$.fragment,e),Ut=r(e),je=s(e,"DIV",{class:!0});var Wo=u(je);g(We.$$.fragment,Wo),Wo.forEach(t),Rt=r(e),Be=s(e,"DIV",{class:!0});var Bo=u(Be);g(Fe.$$.fragment,Bo),Bo.forEach(t),Ht=r(e),g(Oe.$$.fragment,e),St=r(e),C=s(e,"DIV",{class:!0});var G=u(C);g(Ze.$$.fragment,G),Qn=r(G),bt=s(G,"P",{"data-svelte-h":!0}),x(bt)!=="svelte-1bspzi1"&&(bt.textContent=Co),Kn=r(G),$t=s(G,"P",{"data-svelte-h":!0}),x($t)!=="svelte-12bfx7b"&&($t.textContent=ko),eo=r(G),yt=s(G,"P",{"data-svelte-h":!0}),x(yt)!=="svelte-3nnpa5"&&(yt.textContent=Io),to=r(G),g(oe.$$.fragment,G),no=r(G),xt=s(G,"P",{"data-svelte-h":!0}),x(xt)!=="svelte-1w0ij7s"&&(xt.innerHTML=Lo),oo=r(G),Mt=s(G,"P",{"data-svelte-h":!0}),x(Mt)!=="svelte-1biq3pv"&&(Mt.textContent=Do),ro=r(G),Nt=s(G,"UL",{"data-svelte-h":!0}),x(Nt)!=="svelte-1x1r32d"&&(Nt.innerHTML=Eo),G.forEach(t),Xt=r(e),g(Ue.$$.fragment,e),Yt=r(e),O=s(e,"DIV",{class:!0});var bn=u(O);g(Re.$$.fragment,bn),lo=r(bn),re=s(bn,"DIV",{class:!0});var $n=u(re);g(He.$$.fragment,$n),ao=r($n),wt=s($n,"P",{"data-svelte-h":!0}),x(wt)!=="svelte-1smaw30"&&(wt.textContent=Go),$n.forEach(t),bn.forEach(t),Qt=r(e),g(Se.$$.fragment,e),Kt=r(e),g(Xe.$$.fragment,e),en=r(e),Ye=s(e,"DIV",{class:!0});var Fo=u(Ye);g(Qe.$$.fragment,Fo),Fo.forEach(t),tn=r(e),Z=s(e,"DIV",{class:!0});var yn=u(Z);g(Ke.$$.fragment,yn),so=r(yn),le=s(yn,"DIV",{class:!0});var xn=u(le);g(et.$$.fragment,xn),io=r(xn),Tt=s(xn,"P",{"data-svelte-h":!0}),x(Tt)!=="svelte-1smaw30"&&(Tt.textContent=qo),xn.forEach(t),yn.forEach(t),nn=r(e),g(tt.$$.fragment,e),on=r(e),It=s(e,"P",{}),u(It).forEach(t),this.h()},h(){h(v,"name","hf:doc:metadata"),h(v,"content",cr),h(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(ee,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(ne,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(Me,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(we,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(ke,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(Le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(qe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(Ae,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(je,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(Be,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(re,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(Ye,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,l){n(document.head,v),i(e,w,l),i(e,D,l),i(e,M,l),c(T,e,l),i(e,$,l),c(N,e,l),i(e,P,l),c(k,e,l),i(e,j,l),i(e,_,l),c(y,_,null),n(_,q),n(_,J),c(W,J,null),n(J,Mn),n(J,nt),n(_,Nn),n(_,U),c(ae,U,null),n(U,wn),n(U,ot),n(_,Tn),n(_,R),c(se,R,null),n(R,Cn),n(R,rt),n(_,kn),n(_,H),c(ie,H,null),n(H,In),n(H,lt),n(_,Ln),n(_,S),c(de,S,null),n(S,Dn),n(S,at),n(_,En),n(_,X),c(me,X,null),n(X,Gn),n(X,st),n(_,qn),n(_,B),c(pe,B,null),n(B,zn),n(B,it),n(B,An),n(B,dt),i(e,Lt,l),c(ge,e,l),i(e,Dt,l),c(ce,e,l),i(e,Et,l),i(e,V,l),c(fe,V,null),n(V,Jn),n(V,mt),n(V,Vn),n(V,pt),i(e,Gt,l),i(e,I,l),c(he,I,null),n(I,Pn),n(I,Y),c(ue,Y,null),n(Y,jn),n(Y,gt),n(I,Wn),n(I,Q),c(_e,Q,null),n(Q,Bn),n(Q,ct),n(I,Fn),n(I,K),c(ve,K,null),n(K,On),n(K,ft),n(I,Zn),n(I,ee),c(be,ee,null),n(ee,Un),n(ee,ht),n(I,Rn),n(I,te),c($e,te,null),n(te,Hn),n(te,ut),n(I,Sn),n(I,ne),c(ye,ne,null),n(ne,Xn),n(ne,_t),i(e,qt,l),c(xe,e,l),i(e,zt,l),i(e,Me,l),c(Ne,Me,null),i(e,At,l),i(e,we,l),c(Te,we,null),i(e,Jt,l),c(Ce,e,l),i(e,Vt,l),i(e,ke,l),c(Ie,ke,null),i(e,Pt,l),i(e,Le,l),c(De,Le,null),i(e,jt,l),c(Ee,e,l),i(e,Wt,l),c(Ge,e,l),i(e,Bt,l),i(e,qe,l),c(ze,qe,null),i(e,Ft,l),i(e,Ae,l),c(Je,Ae,null),i(e,Ot,l),i(e,F,l),c(Ve,F,null),n(F,Yn),n(F,vt),i(e,Zt,l),c(Pe,e,l),i(e,Ut,l),i(e,je,l),c(We,je,null),i(e,Rt,l),i(e,Be,l),c(Fe,Be,null),i(e,Ht,l),c(Oe,e,l),i(e,St,l),i(e,C,l),c(Ze,C,null),n(C,Qn),n(C,bt),n(C,Kn),n(C,$t),n(C,eo),n(C,yt),n(C,to),c(oe,C,null),n(C,no),n(C,xt),n(C,oo),n(C,Mt),n(C,ro),n(C,Nt),i(e,Xt,l),c(Ue,e,l),i(e,Yt,l),i(e,O,l),c(Re,O,null),n(O,lo),n(O,re),c(He,re,null),n(re,ao),n(re,wt),i(e,Qt,l),c(Se,e,l),i(e,Kt,l),c(Xe,e,l),i(e,en,l),i(e,Ye,l),c(Qe,Ye,null),i(e,tn,l),i(e,Z,l),c(Ke,Z,null),n(Z,so),n(Z,le),c(et,le,null),n(le,io),n(le,Tt),i(e,nn,l),c(tt,e,l),i(e,on,l),i(e,It,l),rn=!0},p(e,[l]){const L={};l&2&&(L.$$scope={dirty:l,ctx:e}),oe.$set(L)},i(e){rn||(d(T.$$.fragment,e),d(N.$$.fragment,e),d(k.$$.fragment,e),d(y.$$.fragment,e),d(W.$$.fragment,e),d(ae.$$.fragment,e),d(se.$$.fragment,e),d(ie.$$.fragment,e),d(de.$$.fragment,e),d(me.$$.fragment,e),d(pe.$$.fragment,e),d(ge.$$.fragment,e),d(ce.$$.fragment,e),d(fe.$$.fragment,e),d(he.$$.fragment,e),d(ue.$$.fragment,e),d(_e.$$.fragment,e),d(ve.$$.fragment,e),d(be.$$.fragment,e),d($e.$$.fragment,e),d(ye.$$.fragment,e),d(xe.$$.fragment,e),d(Ne.$$.fragment,e),d(Te.$$.fragment,e),d(Ce.$$.fragment,e),d(Ie.$$.fragment,e),d(De.$$.fragment,e),d(Ee.$$.fragment,e),d(Ge.$$.fragment,e),d(ze.$$.fragment,e),d(Je.$$.fragment,e),d(Ve.$$.fragment,e),d(Pe.$$.fragment,e),d(We.$$.fragment,e),d(Fe.$$.fragment,e),d(Oe.$$.fragment,e),d(Ze.$$.fragment,e),d(oe.$$.fragment,e),d(Ue.$$.fragment,e),d(Re.$$.fragment,e),d(He.$$.fragment,e),d(Se.$$.fragment,e),d(Xe.$$.fragment,e),d(Qe.$$.fragment,e),d(Ke.$$.fragment,e),d(et.$$.fragment,e),d(tt.$$.fragment,e),rn=!0)},o(e){m(T.$$.fragment,e),m(N.$$.fragment,e),m(k.$$.fragment,e),m(y.$$.fragment,e),m(W.$$.fragment,e),m(ae.$$.fragment,e),m(se.$$.fragment,e),m(ie.$$.fragment,e),m(de.$$.fragment,e),m(me.$$.fragment,e),m(pe.$$.fragment,e),m(ge.$$.fragment,e),m(ce.$$.fragment,e),m(fe.$$.fragment,e),m(he.$$.fragment,e),m(ue.$$.fragment,e),m(_e.$$.fragment,e),m(ve.$$.fragment,e),m(be.$$.fragment,e),m($e.$$.fragment,e),m(ye.$$.fragment,e),m(xe.$$.fragment,e),m(Ne.$$.fragment,e),m(Te.$$.fragment,e),m(Ce.$$.fragment,e),m(Ie.$$.fragment,e),m(De.$$.fragment,e),m(Ee.$$.fragment,e),m(Ge.$$.fragment,e),m(ze.$$.fragment,e),m(Je.$$.fragment,e),m(Ve.$$.fragment,e),m(Pe.$$.fragment,e),m(We.$$.fragment,e),m(Fe.$$.fragment,e),m(Oe.$$.fragment,e),m(Ze.$$.fragment,e),m(oe.$$.fragment,e),m(Ue.$$.fragment,e),m(Re.$$.fragment,e),m(He.$$.fragment,e),m(Se.$$.fragment,e),m(Xe.$$.fragment,e),m(Qe.$$.fragment,e),m(Ke.$$.fragment,e),m(et.$$.fragment,e),m(tt.$$.fragment,e),rn=!1},d(e){e&&(t(w),t(D),t(M),t($),t(P),t(j),t(_),t(Lt),t(Dt),t(Et),t(V),t(Gt),t(I),t(qt),t(zt),t(Me),t(At),t(we),t(Jt),t(Vt),t(ke),t(Pt),t(Le),t(jt),t(Wt),t(Bt),t(qe),t(Ft),t(Ae),t(Ot),t(F),t(Zt),t(Ut),t(je),t(Rt),t(Be),t(Ht),t(St),t(C),t(Xt),t(Yt),t(O),t(Qt),t(Kt),t(en),t(Ye),t(tn),t(Z),t(nn),t(on),t(It)),t(v),f(T,e),f(N,e),f(k,e),f(y),f(W),f(ae),f(se),f(ie),f(de),f(me),f(pe),f(ge,e),f(ce,e),f(fe),f(he),f(ue),f(_e),f(ve),f(be),f($e),f(ye),f(xe,e),f(Ne),f(Te),f(Ce,e),f(Ie),f(De),f(Ee,e),f(Ge,e),f(ze),f(Je),f(Ve),f(Pe,e),f(We),f(Fe),f(Oe,e),f(Ze),f(oe),f(Ue,e),f(Re),f(He),f(Se,e),f(Xe,e),f(Qe),f(Ke),f(et),f(tt,e)}}}const cr='{"title":"Models","local":"models","sections":[{"title":"Model","local":"model","sections":[{"title":"LightevalModel","local":"lighteval.models.abstract_model.LightevalModel","sections":[],"depth":3}],"depth":2},{"title":"Accelerate and Transformers Models","local":"accelerate-and-transformers-models","sections":[{"title":"TransformersModel","local":"lighteval.models.transformers.transformers_model.TransformersModelConfig","sections":[],"depth":3},{"title":"AdapterModel","local":"lighteval.models.transformers.adapter_model.AdapterModelConfig","sections":[],"depth":3},{"title":"DeltaModel","local":"lighteval.models.transformers.delta_model.DeltaModelConfig","sections":[],"depth":3}],"depth":2},{"title":"Endpoints-based Models","local":"endpoints-based-models","sections":[{"title":"InferenceEndpointModel","local":"lighteval.models.endpoints.endpoint_model.InferenceEndpointModelConfig","sections":[],"depth":3},{"title":"TGI ModelClient","local":"lighteval.models.endpoints.tgi_model.TGIModelConfig","sections":[],"depth":3},{"title":"Custom Model","local":"lighteval.models.custom.custom_model.CustomModelConfig","sections":[],"depth":3},{"title":"Open AI Models","local":"lighteval.models.endpoints.openai_model.OpenAIClient","sections":[],"depth":3}],"depth":2},{"title":"VLLM Model","local":"vllm-model","sections":[{"title":"VLLMModel","local":"lighteval.models.vllm.vllm_model.VLLMModelConfig","sections":[],"depth":3}],"depth":2}],"depth":1}';function fr(E){return Uo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $r extends Ro{constructor(v){super(),Ho(this,v,fr,gr,Zo,{})}}export{$r as component};