lighteval/pr_1221/en/_app/immutable/nodes/18.2c812be7.js

import{s as Te,o as Je,n as P}from"../chunks/scheduler.3a17fb72.js";import{S as ve,i as xe,e as m,s as p,c as d,h as we,a as u,d as c,b as i,f as ce,g as h,j as $,k as me,l as r,m as b,n as g,t as U,o as y,p as f}from"../chunks/index.093f8863.js";import{C as ke,H as Re,E as Qe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.5e7ea2bd.js";import{D as Ne}from"../chunks/Docstring.3b8777bb.js";import{C as Y}from"../chunks/CodeBlock.09235327.js";import{E as X}from"../chunks/ExampleCodeBlock.dadac55a.js";function Ee(j){let t,o;return t=new Y({props:{code:"cmVzcG9uc2UlMjAlM0QlMjBNb2RlbFJlc3BvbnNlKCUwQSUyMCUyMCUyMCUyMHRleHQlM0QlNUIlMjJUaGUlMjBjYXBpdGFsJTIwb2YlMjBGcmFuY2UlMjBpcyUyMFBhcmlzLiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGlucHV0X3Rva2VucyUzRCU1QjElMkMlMjAyJTJDJTIwMyUyQyUyMDQlNUQlMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfdG9rZW5zJTNEJTVCJTVCNSUyQyUyMDYlMkMlMjA3JTJDJTIwOCU1RCU1RCUwQSk=",highlighted:`response = ModelResponse(
    text=[<span class="hljs-string">&quot;The capital of France is Paris.&quot;</span>],
    input_tokens=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>],
    output_tokens=[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]
)`,wrap:!1}}),{c(){d(t.$$.fragment)},l(s){h(t.$$.fragment,s)},m(s,M){g(t,s,M),o=!0},p:P,i(s){o||(U(t.$$.fragment,s),o=!0)},o(s){y(t.$$.fragment,s),o=!1},d(s){f(t,s)}}}function Ve(j){let t,o;return t=new Y({props:{code:"cmVzcG9uc2UlMjAlM0QlMjBNb2RlbFJlc3BvbnNlKCUwQSUyMCUyMCUyMCUyMGxvZ3Byb2JzJTNEJTVCLTAuNSUyQyUyMC0xLjIlMkMlMjAtMi4xJTJDJTIwLTEuOCU1RCUyQyUyMCUyMCUyMyUyMExvZ3Byb2JzJTIwZm9yJTIwZWFjaCUyMGNob2ljZSUwQSUyMCUyMCUyMCUyMGFyZ21heF9sb2dpdHNfZXFfZ29sZCUzRCU1QkZhbHNlJTJDJTIwRmFsc2UlMkMlMjBGYWxzZSUyQyUyMEZhbHNlJTVEJTJDJTIwJTIwJTIzJTIwV2hldGhlciUyMGNvcnJlY3QlMjBjaG9pY2UlMjB3YXMlMjBzZWxlY3RlZCUwQSUyMCUyMCUyMCUyMGlucHV0X3Rva2VucyUzRCU1QjElMkMlMjAyJTJDJTIwMyUyQyUyMDQlNUQlMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfdG9rZW5zJTNEJTVCJTVCNSU1RCUyQyUyMCU1QjYlNUQlMkMlMjAlNUI3JTVEJTJDJTIwJTVCOCU1RCU1RCUwQSk=",highlighted:`response = ModelResponse(
    logprobs=[-<span class="hljs-number">0.5</span>, -<span class="hljs-number">1.2</span>, -<span class="hljs-number">2.1</span>, -<span class="hljs-number">1.8</span>],  <span class="hljs-comment"># Logprobs for each choice</span>
    argmax_logits_eq_gold=[<span class="hljs-literal">False</span>, <span class="hljs-literal">False</span>, <span class="hljs-literal">False</span>, <span class="hljs-literal">False</span>],  <span class="hljs-comment"># Whether correct choice was selected</span>
    input_tokens=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>],
    output_tokens=[[<span class="hljs-number">5</span>], [<span class="hljs-number">6</span>], [<span class="hljs-number">7</span>], [<span class="hljs-number">8</span>]]
)`,wrap:!1}}),{c(){d(t.$$.fragment)},l(s){h(t.$$.fragment,s)},m(s,M){g(t,s,M),o=!0},p:P,i(s){o||(U(t.$$.fragment,s),o=!0)},o(s){y(t.$$.fragment,s),o=!1},d(s){f(t,s)}}}function Ae(j){let t,o;return t=new Y({props:{code:"cmVzcG9uc2UlMjAlM0QlMjBNb2RlbFJlc3BvbnNlKCUwQSUyMCUyMCUyMCUyMHRleHQlM0QlNUIlMjJUaGUlMjBtb2RlbCUyMGdlbmVyYXRlZCUyMHRoaXMlMjB0ZXh0LiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGxvZ3Byb2JzJTNEJTVCLTEuMiUyQyUyMC0wLjglMkMlMjAtMS41JTJDJTIwLTAuOSUyQyUyMC0xLjElNUQlMkMlMjAlMjAlMjMlMjBMb2dwcm9icyUyMGZvciUyMGVhY2glMjB0b2tlbiUwQSUyMCUyMCUyMCUyMGlucHV0X3Rva2VucyUzRCU1QjElMkMlMjAyJTJDJTIwMyUyQyUyMDQlMkMlMjA1JTVEJTJDJTBBJTIwJTIwJTIwJTIwb3V0cHV0X3Rva2VucyUzRCU1QiU1QjYlNUQlMkMlMjAlNUI3JTVEJTJDJTIwJTVCOCU1RCUyQyUyMCU1QjklNUQlMkMlMjAlNUIxMCU1RCU1RCUwQSk=",highlighted:`response = ModelResponse(
    text=[<span class="hljs-string">&quot;The model generated this text.&quot;</span>],
    logprobs=[-<span class="hljs-number">1.2</span>, -<span class="hljs-number">0.8</span>, -<span class="hljs-number">1.5</span>, -<span class="hljs-number">0.9</span>, -<span class="hljs-number">1.1</span>],  <span class="hljs-comment"># Logprobs for each token</span>
    input_tokens=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>],
    output_tokens=[[<span class="hljs-number">6</span>], [<span class="hljs-number">7</span>], [<span class="hljs-number">8</span>], [<span class="hljs-number">9</span>], [<span class="hljs-number">10</span>]]
)`,wrap:!1}}),{c(){d(t.$$.fragment)},l(s){h(t.$$.fragment,s)},m(s,M){g(t,s,M),o=!0},p:P,i(s){o||(U(t.$$.fragment,s),o=!0)},o(s){y(t.$$.fragment,s),o=!1},d(s){f(t,s)}}}function Be(j){let t,o;return t=new Y({props:{code:"cmVzcG9uc2UlMjAlM0QlMjBNb2RlbFJlc3BvbnNlKCUwQSUyMCUyMCUyMCUyMHRleHQlM0QlNUIlMjJUaGUlMjBhbnN3ZXIlMjBpcyUyMDQyLiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGxvZ3Byb2JzJTNEJTVCLTEuMSUyQyUyMC0wLjklMkMlMjAtMS4zJTJDJTIwLTAuNyU1RCUyQyUyMCUyMCUyMyUyMENvbmRpdGlvbmVkJTIwbG9ncHJvYnMlMEElMjAlMjAlMjAlMjB1bmNvbmRpdGlvbmVkX2xvZ3Byb2JzJTNEJTVCLTIuMSUyQyUyMC0xLjglMkMlMjAtMi4zJTJDJTIwLTEuNSU1RCUyQyUyMCUyMCUyMyUyMFVuY29uZGl0aW9uZWQlMjBsb2dwcm9icyUwQSUyMCUyMCUyMCUyMGlucHV0X3Rva2VucyUzRCU1QjElMkMlMjAyJTJDJTIwMyUyQyUyMDQlNUQlMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfdG9rZW5zJTNEJTVCJTVCNSU1RCUyQyUyMCU1QjYlNUQlMkMlMjAlNUI3JTVEJTJDJTIwJTVCOCU1RCU1RCUwQSk=",highlighted:`response = ModelResponse(
    text=[<span class="hljs-string">&quot;The answer is 42.&quot;</span>],
    logprobs=[-<span class="hljs-number">1.1</span>, -<span class="hljs-number">0.9</span>, -<span class="hljs-number">1.3</span>, -<span class="hljs-number">0.7</span>],  <span class="hljs-comment"># Conditioned logprobs</span>
    unconditioned_logprobs=[-<span class="hljs-number">2.1</span>, -<span class="hljs-number">1.8</span>, -<span class="hljs-number">2.3</span>, -<span class="hljs-number">1.5</span>],  <span class="hljs-comment"># Unconditioned logprobs</span>
    input_tokens=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>],
    output_tokens=[[<span class="hljs-number">5</span>], [<span class="hljs-number">6</span>], [<span class="hljs-number">7</span>], [<span class="hljs-number">8</span>]]
)`,wrap:!1}}),{c(){d(t.$$.fragment)},l(s){h(t.$$.fragment,s)},m(s,M){g(t,s,M),o=!0},p:P,i(s){o||(U(t.$$.fragment,s),o=!0)},o(s){y(t.$$.fragment,s),o=!1},d(s){f(t,s)}}}function Ie(j){let t,o,s,M,v,H,x,z,w,ue="All models will generate an ouput per Doc supplied to the <code>generation</code> or <code>loglikelihood</code> fuctions.",D,l,k,O,Q,Me="A class to represent the response from a model during evaluation.",W,N,de=`This dataclass contains all the information returned by a model during inference,
including generated text, log probabilities, token information, and metadata.
Different attributes are required for different types of evaluation metrics.`,K,E,he="Usage Examples:",ee,V,ge="<strong>For generative tasks (text completion, summarization):</strong>",se,C,te,A,Ue="<strong>For multiple choice tasks:</strong>",le,_,ne,B,ye="<strong>For perplexity calculation:</strong>",ae,T,oe,I,fe="<strong>For PMI analysis:</strong>",re,J,pe,L,be="Notes:",ie,G,je="<li>For most evaluation tasks, only a subset of attributes is required</li> <li>The <code>text</code> attribute is the most commonly used for generative tasks</li> <li><code>logprobs</code> are essential for probability-based metrics like perplexity</li>",Z,R,F,S,q;return v=new ke({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),x=new Re({props:{title:"Model’s Output",local:"lighteval.models.model_output.ModelResponse",headingTag:"h1"}}),k=new Ne({props:{name:"class lighteval.models.model_output.ModelResponse",anchor:"lighteval.models.model_output.ModelResponse",parameters:[{name:"input",val:": str | list | None = None"},{name:"input_tokens",val:": list = <factory>"},{name:"text",val:": list = <factory>"},{name:"output_tokens",val:": list = <factory>"},{name:"text_post_processed",val:": list[str] | None = None"},{name:"reasonings",val:": list = <factory>"},{name:"logprobs",val:": list = <factory>"},{name:"argmax_logits_eq_gold",val:": list = <factory>"},{name:"logits",val:": list[list[float]] | None = None"},{name:"unconditioned_logprobs",val:": list[float] | None = None"},{name:"truncated_tokens_count",val:": int = 0"},{name:"padded_tokens_count",val:": int = 0"}],parametersDescription:[{anchor:"lighteval.models.model_output.ModelResponse.input",description:`<strong>input</strong> (str | list | None) &#x2014;
The original input prompt or context that was fed to the model.
Used for debugging and analysis purposes.`,name:"input"},{anchor:"lighteval.models.model_output.ModelResponse.input_tokens",description:`<strong>input_tokens</strong> (list[int]) &#x2014;
The tokenized representation of the input prompt.
Useful for understanding how the model processes the input.`,name:"input_tokens"},{anchor:"lighteval.models.model_output.ModelResponse.text",description:`<strong>text</strong> (list[str]) &#x2014;
The generated text responses from the model. Each element represents
one generation (useful when num_samples &gt; 1).
<strong>Required for</strong>: Generative metrics, exact match, llm as a judge, etc.`,name:"text"},{anchor:"lighteval.models.model_output.ModelResponse.text_post_processed",description:`<strong>text_post_processed</strong> (Optional[list[str]]) &#x2014;
The generated text responses from the model, but post processed.
Atm, post processing removes thinking/reasoning steps.</p>
<p>Careful! This is not computed by default, but in a separate step by calling
<code>post_process</code> on the ModelResponse object.
<strong>Required for</strong>: Generative metrics that require direct answers.`,name:"text_post_processed"},{anchor:"lighteval.models.model_output.ModelResponse.logprobs",description:`<strong>logprobs</strong> (list[float]) &#x2014;
Log probabilities of the generated tokens or sequences.
<strong>Required for</strong>: loglikelihood and perplexity metrics.`,name:"logprobs"},{anchor:"lighteval.models.model_output.ModelResponse.argmax_logits_eq_gold",description:`<strong>argmax_logits_eq_gold</strong> (list[bool]) &#x2014;
Whether the argmax logits match the gold/expected text.
Used for accuracy calculations in multiple choice and classification tasks.
<strong>Required for</strong>: certain loglikelihood metrics.`,name:"argmax_logits_eq_gold"}],source:"https://github.com/huggingface/lighteval/blob/vr_1221/src/lighteval/models/model_output.py#L29"}}),C=new X({props:{anchor:"lighteval.models.model_output.ModelResponse.example",$$slots:{default:[Ee]},$$scope:{ctx:j}}}),_=new X({props:{anchor:"lighteval.models.model_output.ModelResponse.example-2",$$slots:{default:[Ve]},$$scope:{ctx:j}}}),T=new X({props:{anchor:"lighteval.models.model_output.ModelResponse.example-3",$$slots:{default:[Ae]},$$scope:{ctx:j}}}),J=new X({props:{anchor:"lighteval.models.model_output.ModelResponse.example-4",$$slots:{default:[Be]},$$scope:{ctx:j}}}),R=new Qe({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/package_reference/models_outputs.mdx"}}),{c(){t=m("meta"),o=p(),s=m("p"),M=p(),d(v.$$.fragment),H=p(),d(x.$$.fragment),z=p(),w=m("p"),w.innerHTML=ue,D=p(),l=m("div"),d(k.$$.fragment),O=p(),Q=m("p"),Q.textContent=Me,W=p(),N=m("p"),N.textContent=de,K=p(),E=m("p"),E.textContent=he,ee=p(),V=m("p"),V.innerHTML=ge,se=p(),d(C.$$.fragment),te=p(),A=m("p"),A.innerHTML=Ue,le=p(),d(_.$$.fragment),ne=p(),B=m("p"),B.innerHTML=ye,ae=p(),d(T.$$.fragment),oe=p(),I=m("p"),I.innerHTML=fe,re=p(),d(J.$$.fragment),pe=p(),L=m("p"),L.textContent=be,ie=p(),G=m("ul"),G.innerHTML=je,Z=p(),d(R.$$.fragment),F=p(),S=m("p"),this.h()},l(e){const a=we("svelte-u9bgzb",document.head);t=u(a,"META",{name:!0,content:!0}),a.forEach(c),o=i(e),s=u(e,"P",{}),ce(s).forEach(c),M=i(e),h(v.$$.fragment,e),H=i(e),h(x.$$.fragment,e),z=i(e),w=u(e,"P",{"data-svelte-h":!0}),$(w)!=="svelte-uqagb5"&&(w.innerHTML=ue),D=i(e),l=u(e,"DIV",{class:!0});var n=ce(l);h(k.$$.fragment,n),O=i(n),Q=u(n,"P",{"data-svelte-h":!0}),$(Q)!=="svelte-3rmmtp"&&(Q.textContent=Me),W=i(n),N=u(n,"P",{"data-svelte-h":!0}),$(N)!=="svelte-fxvm9g"&&(N.textContent=de),K=i(n),E=u(n,"P",{"data-svelte-h":!0}),$(E)!=="svelte-93h1jg"&&(E.textContent=he),ee=i(n),V=u(n,"P",{"data-svelte-h":!0}),$(V)!=="svelte-vino55"&&(V.innerHTML=ge),se=i(n),h(C.$$.fragment,n),te=i(n),A=u(n,"P",{"data-svelte-h":!0}),$(A)!=="svelte-1mwzp2l"&&(A.innerHTML=Ue),le=i(n),h(_.$$.fragment,n),ne=i(n),B=u(n,"P",{"data-svelte-h":!0}),$(B)!=="svelte-1caf309"&&(B.innerHTML=ye),ae=i(n),h(T.$$.fragment,n),oe=i(n),I=u(n,"P",{"data-svelte-h":!0}),$(I)!=="svelte-r50ws8"&&(I.innerHTML=fe),re=i(n),h(J.$$.fragment,n),pe=i(n),L=u(n,"P",{"data-svelte-h":!0}),$(L)!=="svelte-1biq3pv"&&(L.textContent=be),ie=i(n),G=u(n,"UL",{"data-svelte-h":!0}),$(G)!=="svelte-s8k5hx"&&(G.innerHTML=je),n.forEach(c),Z=i(e),h(R.$$.fragment,e),F=i(e),S=u(e,"P",{}),ce(S).forEach(c),this.h()},h(){me(t,"name","hf:doc:metadata"),me(t,"content",Le),me(l,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,a){r(document.head,t),b(e,o,a),b(e,s,a),b(e,M,a),g(v,e,a),b(e,H,a),g(x,e,a),b(e,z,a),b(e,w,a),b(e,D,a),b(e,l,a),g(k,l,null),r(l,O),r(l,Q),r(l,W),r(l,N),r(l,K),r(l,E),r(l,ee),r(l,V),r(l,se),g(C,l,null),r(l,te),r(l,A),r(l,le),g(_,l,null),r(l,ne),r(l,B),r(l,ae),g(T,l,null),r(l,oe),r(l,I),r(l,re),g(J,l,null),r(l,pe),r(l,L),r(l,ie),r(l,G),b(e,Z,a),g(R,e,a),b(e,F,a),b(e,S,a),q=!0},p(e,[a]){const n={};a&2&&(n.$$scope={dirty:a,ctx:e}),C.$set(n);const $e={};a&2&&($e.$$scope={dirty:a,ctx:e}),_.$set($e);const Ce={};a&2&&(Ce.$$scope={dirty:a,ctx:e}),T.$set(Ce);const _e={};a&2&&(_e.$$scope={dirty:a,ctx:e}),J.$set(_e)},i(e){q||(U(v.$$.fragment,e),U(x.$$.fragment,e),U(k.$$.fragment,e),U(C.$$.fragment,e),U(_.$$.fragment,e),U(T.$$.fragment,e),U(J.$$.fragment,e),U(R.$$.fragment,e),q=!0)},o(e){y(v.$$.fragment,e),y(x.$$.fragment,e),y(k.$$.fragment,e),y(C.$$.fragment,e),y(_.$$.fragment,e),y(T.$$.fragment,e),y(J.$$.fragment,e),y(R.$$.fragment,e),q=!1},d(e){e&&(c(o),c(s),c(M),c(H),c(z),c(w),c(D),c(l),c(Z),c(F),c(S)),c(t),f(v,e),f(x,e),f(k),f(C),f(_),f(T),f(J),f(R,e)}}}const Le='{"title":"Model’s Output","local":"lighteval.models.model_output.ModelResponse","sections":[],"depth":1}';function Ge(j){return Je(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class qe extends ve{constructor(t){super(),xe(this,t,Ge,Ie,Te,{})}}export{qe as component};