lighteval/pr_1003/en/_app/immutable/nodes/8.ff4cfecf.js

import{s as be,n as ye,o as ve}from"../chunks/scheduler.5f3e6389.js";import{S as Me,i as Je,e as d,s as n,c as i,h as Ie,a as c,d as l,b as s,f as we,g as o,j as S,k as Te,l as je,m as a,n as r,t as m,o as u,p}from"../chunks/index.373ab25c.js";import{C as ke,H as g,E as Ee}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.487686c5.js";import{C as ue}from"../chunks/CodeBlock.dfb53165.js";function xe(pe){let f,F,H,V,$,A,h,q,w,ge=`🤗 Lighteval is your all-in-one toolkit for evaluating Large Language Models
(LLMs) across multiple backends with ease. Dive deep into your model’s
performance by saving and exploring detailed, sample-by-sample results to debug
and see how your models stack up.`,N,T,z,b,R,y,fe="Evaluate your models using the most popular and efficient inference backends:",P,v,de=`<li><code>transformers</code>: Evaluate models on CPU or one or more GPUs using <a href="https://github.com/huggingface/transformers" rel="nofollow">🤗
Accelerate</a></li> <li><code>nanotron</code>: Evaluate models in distributed settings using <a href="https://github.com/huggingface/nanotron" rel="nofollow">⚡️
Nanotron</a></li> <li><code>vllm</code>: Evaluate models on one or more GPUs using <a href="https://github.com/vllm-project/vllm" rel="nofollow">🚀
VLLM</a></li> <li><code>custom</code>: Evaluate custom models (can be anything)</li> <li><code>sglang</code>: Evaluate models using <a href="https://github.com/sgl-project/sglang" rel="nofollow">SGLang</a> as backend</li> <li><code>inference-endpoint</code>: Evaluate models using Hugging Face’s <a href="https://huggingface.co/inference-endpoints/dedicated" rel="nofollow">Inference Endpoints API</a></li> <li><code>tgi</code>: Evaluate models using <a href="https://huggingface.co/docs/text-generation-inference/en/index" rel="nofollow">🔗 Text Generation Inference</a> running locally</li> <li><code>litellm</code>: Evaluate models on any compatible API using <a href="https://www.litellm.ai/" rel="nofollow">LiteLLM</a></li> <li><code>inference-providers</code>: Evaluate models using <a href="https://huggingface.co/docs/inference-providers/en/index" rel="nofollow">HuggingFace’s inference providers</a> as backend**: Distributed training and evaluation</li>`,X,M,Y,J,ce="<li><strong>Extensive Task Library</strong>: 1000s pre-built evaluation tasks</li> <li><strong>Custom Task Creation</strong>: Build your own evaluation tasks</li> <li><strong>Flexible Metrics</strong>: Support for custom metrics and scoring</li> <li><strong>Detailed Analysis</strong>: Sample-by-sample results for deep insights</li>",Q,I,D,j,$e=`Customization at your fingertips: create <a href="adding-a-custom-task">new tasks</a>,
<a href="adding-a-new-metric">metrics</a> or <a href="evaluating-a-custom-model">model</a> tailored to your needs, or browse all our existing tasks and metrics.`,K,k,O,E,he="Seamlessly experiment, benchmark, and store your results on the Hugging Face Hub, S3, or locally.",ee,x,te,Z,le,L,ae,U,ne,W,se,B,ie,C,oe,G,re,_,me;return $=new ke({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),h=new g({props:{title:"Lighteval",local:"lighteval",headingTag:"h1"}}),T=new g({props:{title:"Key Features",local:"key-features",headingTag:"h2"}}),b=new g({props:{title:"🚀 Multi-Backend Support",local:"-multi-backend-support",headingTag:"h3"}}),M=new g({props:{title:"📊 Comprehensive Evaluation",local:"-comprehensive-evaluation",headingTag:"h3"}}),I=new g({props:{title:"🔧 Easy Customization",local:"-easy-customization",headingTag:"h3"}}),k=new g({props:{title:"☁️ Seamless Integration",local:"-seamless-integration",headingTag:"h3"}}),x=new g({props:{title:"Quick Start",local:"quick-start",headingTag:"h2"}}),Z=new g({props:{title:"Installation",local:"installation",headingTag:"h3"}}),L=new ue({props:{code:"cGlwJTIwaW5zdGFsbCUyMGxpZ2h0ZXZhbA==",highlighted:"pip install lighteval",wrap:!1}}),U=new g({props:{title:"Basic Usage",local:"basic-usage",headingTag:"h3"}}),W=new ue({props:{code:"JTIzJTIwRXZhbHVhdGUlMjBhJTIwbW9kZWwlMjB1c2luZyUyMFRyYW5zZm9ybWVycyUyMGJhY2tlbmQlMEFsaWdodGV2YWwlMjBhY2NlbGVyYXRlJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybW9kZWxfbmFtZSUzRG9wZW5haS1jb21tdW5pdHklMkZncHQyJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybGVhZGVyYm9hcmQlN0N0cnV0aGZ1bHFhJTNBbWMlN0MwJTIy",highlighted:`<span class="hljs-comment"># Evaluate a model using Transformers backend</span>
lighteval accelerate \\
    <span class="hljs-string">&quot;model_name=openai-community/gpt2&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span>`,wrap:!1}}),B=new g({props:{title:"Save Results",local:"save-results",headingTag:"h3"}}),C=new ue({props:{code:"JTIzJTIwU2F2ZSUyMGxvY2FsbHklMEFsaWdodGV2YWwlMjBhY2NlbGVyYXRlJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybW9kZWxfbmFtZSUzRG9wZW5haS1jb21tdW5pdHklMkZncHQyJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybGVhZGVyYm9hcmQlN0N0cnV0aGZ1bHFhJTNBbWMlN0MwJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1vdXRwdXQtZGlyJTIwLiUyRnJlc3VsdHMlMEElMEElMjMlMjBQdXNoJTIwdG8lMjBIdWdnaW5nJTIwRmFjZSUyMEh1YiUwQWxpZ2h0ZXZhbCUyMGFjY2VsZXJhdGUlMjAlNUMlMEElMjAlMjAlMjAlMjAlMjJtb2RlbF9uYW1lJTNEb3BlbmFpLWNvbW11bml0eSUyRmdwdDIlMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAlMjJsZWFkZXJib2FyZCU3Q3RydXRoZnVscWElM0FtYyU3QzAlMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXB1c2gtdG8taHViJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1yZXN1bHRzLW9yZyUyMHlvdXItdXNlcm5hbWU=",highlighted:`<span class="hljs-comment"># Save locally</span>
lighteval accelerate \\
    <span class="hljs-string">&quot;model_name=openai-community/gpt2&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span> \\
    --output-dir ./results

<span class="hljs-comment"># Push to Hugging Face Hub</span>
lighteval accelerate \\
    <span class="hljs-string">&quot;model_name=openai-community/gpt2&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span> \\
    --push-to-hub \\
    --results-org your-username`,wrap:!1}}),G=new Ee({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/index.mdx"}}),{c(){f=d("meta"),F=n(),H=d("p"),V=n(),i($.$$.fragment),A=n(),i(h.$$.fragment),q=n(),w=d("p"),w.textContent=ge,N=n(),i(T.$$.fragment),z=n(),i(b.$$.fragment),R=n(),y=d("p"),y.textContent=fe,P=n(),v=d("ul"),v.innerHTML=de,X=n(),i(M.$$.fragment),Y=n(),J=d("ul"),J.innerHTML=ce,Q=n(),i(I.$$.fragment),D=n(),j=d("p"),j.innerHTML=$e,K=n(),i(k.$$.fragment),O=n(),E=d("p"),E.textContent=he,ee=n(),i(x.$$.fragment),te=n(),i(Z.$$.fragment),le=n(),i(L.$$.fragment),ae=n(),i(U.$$.fragment),ne=n(),i(W.$$.fragment),se=n(),i(B.$$.fragment),ie=n(),i(C.$$.fragment),oe=n(),i(G.$$.fragment),re=n(),_=d("p"),this.h()},l(e){const t=Ie("svelte-u9bgzb",document.head);f=c(t,"META",{name:!0,content:!0}),t.forEach(l),F=s(e),H=c(e,"P",{}),we(H).forEach(l),V=s(e),o($.$$.fragment,e),A=s(e),o(h.$$.fragment,e),q=s(e),w=c(e,"P",{"data-svelte-h":!0}),S(w)!=="svelte-5jd4gg"&&(w.textContent=ge),N=s(e),o(T.$$.fragment,e),z=s(e),o(b.$$.fragment,e),R=s(e),y=c(e,"P",{"data-svelte-h":!0}),S(y)!=="svelte-hqmq3z"&&(y.textContent=fe),P=s(e),v=c(e,"UL",{"data-svelte-h":!0}),S(v)!=="svelte-t4bk74"&&(v.innerHTML=de),X=s(e),o(M.$$.fragment,e),Y=s(e),J=c(e,"UL",{"data-svelte-h":!0}),S(J)!=="svelte-1kvd8am"&&(J.innerHTML=ce),Q=s(e),o(I.$$.fragment,e),D=s(e),j=c(e,"P",{"data-svelte-h":!0}),S(j)!=="svelte-d94dpm"&&(j.innerHTML=$e),K=s(e),o(k.$$.fragment,e),O=s(e),E=c(e,"P",{"data-svelte-h":!0}),S(E)!=="svelte-12kzs6l"&&(E.textContent=he),ee=s(e),o(x.$$.fragment,e),te=s(e),o(Z.$$.fragment,e),le=s(e),o(L.$$.fragment,e),ae=s(e),o(U.$$.fragment,e),ne=s(e),o(W.$$.fragment,e),se=s(e),o(B.$$.fragment,e),ie=s(e),o(C.$$.fragment,e),oe=s(e),o(G.$$.fragment,e),re=s(e),_=c(e,"P",{}),we(_).forEach(l),this.h()},h(){Te(f,"name","hf:doc:metadata"),Te(f,"content",Ze)},m(e,t){je(document.head,f),a(e,F,t),a(e,H,t),a(e,V,t),r($,e,t),a(e,A,t),r(h,e,t),a(e,q,t),a(e,w,t),a(e,N,t),r(T,e,t),a(e,z,t),r(b,e,t),a(e,R,t),a(e,y,t),a(e,P,t),a(e,v,t),a(e,X,t),r(M,e,t),a(e,Y,t),a(e,J,t),a(e,Q,t),r(I,e,t),a(e,D,t),a(e,j,t),a(e,K,t),r(k,e,t),a(e,O,t),a(e,E,t),a(e,ee,t),r(x,e,t),a(e,te,t),r(Z,e,t),a(e,le,t),r(L,e,t),a(e,ae,t),r(U,e,t),a(e,ne,t),r(W,e,t),a(e,se,t),r(B,e,t),a(e,ie,t),r(C,e,t),a(e,oe,t),r(G,e,t),a(e,re,t),a(e,_,t),me=!0},p:ye,i(e){me||(m($.$$.fragment,e),m(h.$$.fragment,e),m(T.$$.fragment,e),m(b.$$.fragment,e),m(M.$$.fragment,e),m(I.$$.fragment,e),m(k.$$.fragment,e),m(x.$$.fragment,e),m(Z.$$.fragment,e),m(L.$$.fragment,e),m(U.$$.fragment,e),m(W.$$.fragment,e),m(B.$$.fragment,e),m(C.$$.fragment,e),m(G.$$.fragment,e),me=!0)},o(e){u($.$$.fragment,e),u(h.$$.fragment,e),u(T.$$.fragment,e),u(b.$$.fragment,e),u(M.$$.fragment,e),u(I.$$.fragment,e),u(k.$$.fragment,e),u(x.$$.fragment,e),u(Z.$$.fragment,e),u(L.$$.fragment,e),u(U.$$.fragment,e),u(W.$$.fragment,e),u(B.$$.fragment,e),u(C.$$.fragment,e),u(G.$$.fragment,e),me=!1},d(e){e&&(l(F),l(H),l(V),l(A),l(q),l(w),l(N),l(z),l(R),l(y),l(P),l(v),l(X),l(Y),l(J),l(Q),l(D),l(j),l(K),l(O),l(E),l(ee),l(te),l(le),l(ae),l(ne),l(se),l(ie),l(oe),l(re),l(_)),l(f),p($,e),p(h,e),p(T,e),p(b,e),p(M,e),p(I,e),p(k,e),p(x,e),p(Z,e),p(L,e),p(U,e),p(W,e),p(B,e),p(C,e),p(G,e)}}}const Ze='{"title":"Lighteval","local":"lighteval","sections":[{"title":"Key Features","local":"key-features","sections":[{"title":"🚀 Multi-Backend Support","local":"-multi-backend-support","sections":[],"depth":3},{"title":"📊 Comprehensive Evaluation","local":"-comprehensive-evaluation","sections":[],"depth":3},{"title":"🔧 Easy Customization","local":"-easy-customization","sections":[],"depth":3},{"title":"☁️ Seamless Integration","local":"-seamless-integration","sections":[],"depth":3}],"depth":2},{"title":"Quick Start","local":"quick-start","sections":[{"title":"Installation","local":"installation","sections":[],"depth":3},{"title":"Basic Usage","local":"basic-usage","sections":[],"depth":3},{"title":"Save Results","local":"save-results","sections":[],"depth":3}],"depth":2}],"depth":1}';function Le(pe){return ve(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ge extends Me{constructor(f){super(),Je(this,f,Le,xe,be,{})}}export{Ge as component};