lighteval/pr_985/en/_app/immutable/nodes/8.628334fa.js

import{s as we,n as Te,o as be}from"../chunks/scheduler.7da89386.js";import{S as ye,i as ve,g as d,s as n,r as i,A as Je,h as c,f as l,c as s,j as $e,u as o,x as C,k as he,y as Me,a,v as r,d as m,t as u,w as g}from"../chunks/index.20910acc.js";import{C as re}from"../chunks/CodeBlock.143bd81e.js";import{H as p,E as Ie}from"../chunks/getInferenceSnippets.7cf363b6.js";function je(me){let f,_,H,F,$,A,h,ue=`🤗 Lighteval is your all-in-one toolkit for evaluating Large Language Models
(LLMs) across multiple backends with ease. Dive deep into your model’s
performance by saving and exploring detailed, sample-by-sample results to debug
and see how your models stack up.`,V,w,q,T,N,b,ge="Evaluate your models using the most popular and efficient inference backends:",R,y,pe=`<li><code>transformers</code>: Evaluate models on CPU or one or more GPUs using <a href="https://github.com/huggingface/transformers" rel="nofollow">🤗
Accelerate</a></li> <li><code>nanotron</code>: Evaluate models in distributed settings using <a href="https://github.com/huggingface/nanotron" rel="nofollow">⚡️
Nanotron</a></li> <li><code>vllm</code>: Evaluate models on one or more GPUs using <a href="https://github.com/vllm-project/vllm" rel="nofollow">🚀
VLLM</a></li> <li><code>custom</code>: Evaluate custom models (can be anything)</li> <li><code>sglang</code>: Evaluate models using <a href="https://github.com/sgl-project/sglang" rel="nofollow">SGLang</a> as backend</li> <li><code>inference-endpoint</code>: Evaluate models using Hugging Face’s <a href="https://huggingface.co/inference-endpoints/dedicated" rel="nofollow">Inference Endpoints API</a></li> <li><code>tgi</code>: Evaluate models using <a href="https://huggingface.co/docs/text-generation-inference/en/index" rel="nofollow">🔗 Text Generation Inference</a> running locally</li> <li><code>litellm</code>: Evaluate models on any compatible API using <a href="https://www.litellm.ai/" rel="nofollow">LiteLLM</a></li> <li><code>inference-providers</code>: Evaluate models using <a href="https://huggingface.co/docs/inference-providers/en/index" rel="nofollow">HuggingFace’s inference providers</a> as backend**: Distributed training and evaluation</li>`,z,v,P,J,fe="<li><strong>Extensive Task Library</strong>: 1000s pre-built evaluation tasks</li> <li><strong>Custom Task Creation</strong>: Build your own evaluation tasks</li> <li><strong>Flexible Metrics</strong>: Support for custom metrics and scoring</li> <li><strong>Detailed Analysis</strong>: Sample-by-sample results for deep insights</li>",X,M,Y,I,de=`Customization at your fingertips: create <a href="adding-a-custom-task">new tasks</a>,
<a href="adding-a-new-metric">metrics</a> or <a href="evaluating-a-custom-model">model</a> tailored to your needs, or browse all our existing tasks and metrics.`,Q,j,D,k,ce="Seamlessly experiment, benchmark, and store your results on the Hugging Face Hub, S3, or locally.",K,E,O,x,ee,Z,te,L,le,U,ae,W,ne,B,se,G,ie,S,oe;return $=new p({props:{title:"Lighteval",local:"lighteval",headingTag:"h1"}}),w=new p({props:{title:"Key Features",local:"key-features",headingTag:"h2"}}),T=new p({props:{title:"🚀 Multi-Backend Support",local:"-multi-backend-support",headingTag:"h3"}}),v=new p({props:{title:"📊 Comprehensive Evaluation",local:"-comprehensive-evaluation",headingTag:"h3"}}),M=new p({props:{title:"🔧 Easy Customization",local:"-easy-customization",headingTag:"h3"}}),j=new p({props:{title:"☁️ Seamless Integration",local:"-seamless-integration",headingTag:"h3"}}),E=new p({props:{title:"Quick Start",local:"quick-start",headingTag:"h2"}}),x=new p({props:{title:"Installation",local:"installation",headingTag:"h3"}}),Z=new re({props:{code:"cGlwJTIwaW5zdGFsbCUyMGxpZ2h0ZXZhbA==",highlighted:"pip install lighteval",wrap:!1}}),L=new p({props:{title:"Basic Usage",local:"basic-usage",headingTag:"h3"}}),U=new re({props:{code:"JTIzJTIwRXZhbHVhdGUlMjBhJTIwbW9kZWwlMjB1c2luZyUyMFRyYW5zZm9ybWVycyUyMGJhY2tlbmQlMEFsaWdodGV2YWwlMjBhY2NlbGVyYXRlJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybW9kZWxfbmFtZSUzRG9wZW5haS1jb21tdW5pdHklMkZncHQyJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybGVhZGVyYm9hcmQlN0N0cnV0aGZ1bHFhJTNBbWMlN0MwJTIy",highlighted:`<span class="hljs-comment"># Evaluate a model using Transformers backend</span>
lighteval accelerate \\
    <span class="hljs-string">&quot;model_name=openai-community/gpt2&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span>`,wrap:!1}}),W=new p({props:{title:"Save Results",local:"save-results",headingTag:"h3"}}),B=new re({props:{code:"JTIzJTIwU2F2ZSUyMGxvY2FsbHklMEFsaWdodGV2YWwlMjBhY2NlbGVyYXRlJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybW9kZWxfbmFtZSUzRG9wZW5haS1jb21tdW5pdHklMkZncHQyJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybGVhZGVyYm9hcmQlN0N0cnV0aGZ1bHFhJTNBbWMlN0MwJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1vdXRwdXQtZGlyJTIwLiUyRnJlc3VsdHMlMEElMEElMjMlMjBQdXNoJTIwdG8lMjBIdWdnaW5nJTIwRmFjZSUyMEh1YiUwQWxpZ2h0ZXZhbCUyMGFjY2VsZXJhdGUlMjAlNUMlMEElMjAlMjAlMjAlMjAlMjJtb2RlbF9uYW1lJTNEb3BlbmFpLWNvbW11bml0eSUyRmdwdDIlMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAlMjJsZWFkZXJib2FyZCU3Q3RydXRoZnVscWElM0FtYyU3QzAlMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXB1c2gtdG8taHViJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1yZXN1bHRzLW9yZyUyMHlvdXItdXNlcm5hbWU=",highlighted:`<span class="hljs-comment"># Save locally</span>
lighteval accelerate \\
    <span class="hljs-string">&quot;model_name=openai-community/gpt2&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span> \\
    --output-dir ./results

<span class="hljs-comment"># Push to Hugging Face Hub</span>
lighteval accelerate \\
    <span class="hljs-string">&quot;model_name=openai-community/gpt2&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span> \\
    --push-to-hub \\
    --results-org your-username`,wrap:!1}}),G=new Ie({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/index.mdx"}}),{c(){f=d("meta"),_=n(),H=d("p"),F=n(),i($.$$.fragment),A=n(),h=d("p"),h.textContent=ue,V=n(),i(w.$$.fragment),q=n(),i(T.$$.fragment),N=n(),b=d("p"),b.textContent=ge,R=n(),y=d("ul"),y.innerHTML=pe,z=n(),i(v.$$.fragment),P=n(),J=d("ul"),J.innerHTML=fe,X=n(),i(M.$$.fragment),Y=n(),I=d("p"),I.innerHTML=de,Q=n(),i(j.$$.fragment),D=n(),k=d("p"),k.textContent=ce,K=n(),i(E.$$.fragment),O=n(),i(x.$$.fragment),ee=n(),i(Z.$$.fragment),te=n(),i(L.$$.fragment),le=n(),i(U.$$.fragment),ae=n(),i(W.$$.fragment),ne=n(),i(B.$$.fragment),se=n(),i(G.$$.fragment),ie=n(),S=d("p"),this.h()},l(e){const t=Je("svelte-u9bgzb",document.head);f=c(t,"META",{name:!0,content:!0}),t.forEach(l),_=s(e),H=c(e,"P",{}),$e(H).forEach(l),F=s(e),o($.$$.fragment,e),A=s(e),h=c(e,"P",{"data-svelte-h":!0}),C(h)!=="svelte-5jd4gg"&&(h.textContent=ue),V=s(e),o(w.$$.fragment,e),q=s(e),o(T.$$.fragment,e),N=s(e),b=c(e,"P",{"data-svelte-h":!0}),C(b)!=="svelte-hqmq3z"&&(b.textContent=ge),R=s(e),y=c(e,"UL",{"data-svelte-h":!0}),C(y)!=="svelte-t4bk74"&&(y.innerHTML=pe),z=s(e),o(v.$$.fragment,e),P=s(e),J=c(e,"UL",{"data-svelte-h":!0}),C(J)!=="svelte-1kvd8am"&&(J.innerHTML=fe),X=s(e),o(M.$$.fragment,e),Y=s(e),I=c(e,"P",{"data-svelte-h":!0}),C(I)!=="svelte-d94dpm"&&(I.innerHTML=de),Q=s(e),o(j.$$.fragment,e),D=s(e),k=c(e,"P",{"data-svelte-h":!0}),C(k)!=="svelte-12kzs6l"&&(k.textContent=ce),K=s(e),o(E.$$.fragment,e),O=s(e),o(x.$$.fragment,e),ee=s(e),o(Z.$$.fragment,e),te=s(e),o(L.$$.fragment,e),le=s(e),o(U.$$.fragment,e),ae=s(e),o(W.$$.fragment,e),ne=s(e),o(B.$$.fragment,e),se=s(e),o(G.$$.fragment,e),ie=s(e),S=c(e,"P",{}),$e(S).forEach(l),this.h()},h(){he(f,"name","hf:doc:metadata"),he(f,"content",ke)},m(e,t){Me(document.head,f),a(e,_,t),a(e,H,t),a(e,F,t),r($,e,t),a(e,A,t),a(e,h,t),a(e,V,t),r(w,e,t),a(e,q,t),r(T,e,t),a(e,N,t),a(e,b,t),a(e,R,t),a(e,y,t),a(e,z,t),r(v,e,t),a(e,P,t),a(e,J,t),a(e,X,t),r(M,e,t),a(e,Y,t),a(e,I,t),a(e,Q,t),r(j,e,t),a(e,D,t),a(e,k,t),a(e,K,t),r(E,e,t),a(e,O,t),r(x,e,t),a(e,ee,t),r(Z,e,t),a(e,te,t),r(L,e,t),a(e,le,t),r(U,e,t),a(e,ae,t),r(W,e,t),a(e,ne,t),r(B,e,t),a(e,se,t),r(G,e,t),a(e,ie,t),a(e,S,t),oe=!0},p:Te,i(e){oe||(m($.$$.fragment,e),m(w.$$.fragment,e),m(T.$$.fragment,e),m(v.$$.fragment,e),m(M.$$.fragment,e),m(j.$$.fragment,e),m(E.$$.fragment,e),m(x.$$.fragment,e),m(Z.$$.fragment,e),m(L.$$.fragment,e),m(U.$$.fragment,e),m(W.$$.fragment,e),m(B.$$.fragment,e),m(G.$$.fragment,e),oe=!0)},o(e){u($.$$.fragment,e),u(w.$$.fragment,e),u(T.$$.fragment,e),u(v.$$.fragment,e),u(M.$$.fragment,e),u(j.$$.fragment,e),u(E.$$.fragment,e),u(x.$$.fragment,e),u(Z.$$.fragment,e),u(L.$$.fragment,e),u(U.$$.fragment,e),u(W.$$.fragment,e),u(B.$$.fragment,e),u(G.$$.fragment,e),oe=!1},d(e){e&&(l(_),l(H),l(F),l(A),l(h),l(V),l(q),l(N),l(b),l(R),l(y),l(z),l(P),l(J),l(X),l(Y),l(I),l(Q),l(D),l(k),l(K),l(O),l(ee),l(te),l(le),l(ae),l(ne),l(se),l(ie),l(S)),l(f),g($,e),g(w,e),g(T,e),g(v,e),g(M,e),g(j,e),g(E,e),g(x,e),g(Z,e),g(L,e),g(U,e),g(W,e),g(B,e),g(G,e)}}}const ke='{"title":"Lighteval","local":"lighteval","sections":[{"title":"Key Features","local":"key-features","sections":[{"title":"🚀 Multi-Backend Support","local":"-multi-backend-support","sections":[],"depth":3},{"title":"📊 Comprehensive Evaluation","local":"-comprehensive-evaluation","sections":[],"depth":3},{"title":"🔧 Easy Customization","local":"-easy-customization","sections":[],"depth":3},{"title":"☁️ Seamless Integration","local":"-seamless-integration","sections":[],"depth":3}],"depth":2},{"title":"Quick Start","local":"quick-start","sections":[{"title":"Installation","local":"installation","sections":[],"depth":3},{"title":"Basic Usage","local":"basic-usage","sections":[],"depth":3},{"title":"Save Results","local":"save-results","sections":[],"depth":3}],"depth":2}],"depth":1}';function Ee(me){return be(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class We extends ye{constructor(f){super(),ve(this,f,Ee,je,we,{})}}export{We as component};