lighteval/pr_476/en/_app/immutable/nodes/18.e032caa7.js

import{s as Be,o as ke,n as _e}from"../chunks/scheduler.7da89386.js";import{S as Ae,i as Ze,g as i,s as a,r,A as xe,h as m,f as l,c as n,j as ve,u as c,x as o,k as Ce,y as Ee,a as s,v as u,d as h,t as f,w}from"../chunks/index.0b7befd3.js";import{T as Ge}from"../chunks/Tip.1e71740f.js";import{C as ce}from"../chunks/CodeBlock.ce33a881.js";import{H as ue,E as Ve}from"../chunks/EditOnGithub.0cb2bc8e.js";function Se(H){let p,T=`Do not forget to prepend the provider in the <code>model_name</code>. Here we use an
openai compatible endpoint to the provider is <code>openai</code>.`;return{c(){p=i("p"),p.innerHTML=T},l(M){p=m(M,"P",{"data-svelte-h":!0}),o(p)!=="svelte-hnl7ld"&&(p.innerHTML=T)},m(M,X){s(M,p,X)},p:_e,d(M){M&&l(p)}}}function We(H){let p,T,M,X,J,Q,g,he=`Lighteval allows to use litellm, a backend allowing you to call all LLM APIs
using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure,
OpenAI, Groq etc.].`,L,d,fe='Documentation for available APIs and compatible endpoints can be found <a href="https://docs.litellm.ai/docs/" rel="nofollow">here</a>.',N,y,q,U,P,b,D,I,we=`Litellm allows generation with any OpenAI compatible endpoint, for example you
can evaluate a model running on a local vllm server.`,F,$,je="To do so you will need to use a config file like so:",z,v,Y,C,O,B,Te="With this you can also access HuggingFace Inference servers, let’s look at how to evaluate DeepSeek-R1-Distill-Qwen-32B.",K,k,Je='First, let’s look at how to acess the model, we can find this from <a href="https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" rel="nofollow">the model card</a>.',ee,_,ge="Step 1:",te,A,de='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lighteval/litellm-guide-2.png" alt="Step 1"/>',le,Z,ye="Step 2:",se,x,Ue='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lighteval/litellm-guide-1.png" alt="Step 2"/>',ae,E,be="Great ! Now we can simply copy paste the base_url and our api key to eval our model.",ne,j,pe,G,ie,V,Ie="And then, we are able to eval our model on any eval available in Lighteval.",me,S,oe,W,Me,R,re;return J=new ue({props:{title:"Litellm as backend",local:"litellm-as-backend",headingTag:"h1"}}),y=new ue({props:{title:"Quick use",local:"quick-use",headingTag:"h2"}}),U=new ce({props:{code:"bGlnaHRldmFsJTIwZW5kcG9pbnQlMjBsaXRlbGxtJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIyZ3B0LTMuNS10dXJibyUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMmxpZ2h0ZXZhbCU3Q2dzbThrJTdDMCU3QzAlMjI=",highlighted:`lighteval endpoint litellm \\
    <span class="hljs-string">&quot;gpt-3.5-turbo&quot;</span> \\
    <span class="hljs-string">&quot;lighteval|gsm8k|0|0&quot;</span>`,wrap:!1}}),b=new ue({props:{title:"Using a config file",local:"using-a-config-file",headingTag:"h2"}}),v=new ce({props:{code:"bW9kZWwlM0ElMEElMjAlMjBiYXNlX3BhcmFtcyUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0ElMjAlMjJvcGVuYWklMkZkZWVwc2Vlay1haSUyRkRlZXBTZWVrLVIxLURpc3RpbGwtUXdlbi0zMkIlMjIlMEElMjAlMjAlMjAlMjBiYXNlX3VybCUzQSUyMCUyMlVSTCUyME9GJTIwVEhFJTIwRU5EUE9JTlQlMjBZT1UlMjBXQU5UJTIwVE8lMjBVU0UlMjIlMEElMjAlMjAlMjAlMjBhcGlfa2V5JTNBJTIwJTIyJTIyJTIwJTIzJTIwcmVtb3ZlJTIwb3IlMjBrZWVwJTIwZW1wdHklMjBhcyUyMG5lZWRlZCUwQSUyMCUyMGdlbmVyYXRpb24lM0ElMEElMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzQSUyMDAuNSUwQSUyMCUyMCUyMCUyMG1heF9uZXdfdG9rZW5zJTNBJTIwMjU2JTBBJTIwJTIwJTIwJTIwc3RvcF90b2tlbnMlM0ElMjAlNUIlMjIlMjIlNUQlMEElMjAlMjAlMjAlMjB0b3BfcCUzQSUyMDAuOSUwQSUyMCUyMCUyMCUyMHNlZWQlM0ElMjAwJTBBJTIwJTIwJTIwJTIwcmVwZXRpdGlvbl9wZW5hbHR5JTNBJTIwMS4wJTBBJTIwJTIwJTIwJTIwZnJlcXVlbmN5X3BlbmFsdHklM0ElMjAwLjA=",highlighted:`<span class="hljs-attr">model:</span>
  <span class="hljs-attr">base_params:</span>
    <span class="hljs-attr">model_name:</span> <span class="hljs-string">&quot;openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B&quot;</span>
    <span class="hljs-attr">base_url:</span> <span class="hljs-string">&quot;URL OF THE ENDPOINT YOU WANT TO USE&quot;</span>
    <span class="hljs-attr">api_key:</span> <span class="hljs-string">&quot;&quot;</span> <span class="hljs-comment"># remove or keep empty as needed</span>
  <span class="hljs-attr">generation:</span>
    <span class="hljs-attr">temperature:</span> <span class="hljs-number">0.5</span>
    <span class="hljs-attr">max_new_tokens:</span> <span class="hljs-number">256</span>
    <span class="hljs-attr">stop_tokens:</span> [<span class="hljs-string">&quot;&quot;</span>]
    <span class="hljs-attr">top_p:</span> <span class="hljs-number">0.9</span>
    <span class="hljs-attr">seed:</span> <span class="hljs-number">0</span>
    <span class="hljs-attr">repetition_penalty:</span> <span class="hljs-number">1.0</span>
    <span class="hljs-attr">frequency_penalty:</span> <span class="hljs-number">0.0</span>`,wrap:!1}}),C=new ue({props:{title:"Use Hugging Face Inference Providers",local:"use-hugging-face-inference-providers",headingTag:"h2"}}),j=new Ge({props:{warning:!0,$$slots:{default:[Se]},$$scope:{ctx:H}}}),G=new ce({props:{code:"bW9kZWwlM0ElMEElMjAlMjBiYXNlX3BhcmFtcyUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0ElMjAlMjJvcGVuYWklMkZkZWVwc2Vlay1haSUyRkRlZXBTZWVrLVIxLURpc3RpbGwtUXdlbi0zMkIlMjIlMEElMjAlMjAlMjAlMjBiYXNlX3VybCUzQSUyMCUyMmh0dHBzJTNBJTJGJTJGcm91dGVyLmh1Z2dpbmdmYWNlLmNvJTJGaGYtaW5mZXJlbmNlJTJGdjElMjIlMEElMjAlMjAlMjAlMjBhcGlfa2V5JTNBJTIwJTIyWU9VUiUyMEtFWSUyMiUyMCUyMyUyMHJlbW92ZSUyMG9yJTIwa2VlcCUyMGVtcHR5JTIwYXMlMjBuZWVkZWQlMEElMjAlMjBnZW5lcmF0aW9uJTNBJTBBJTIwJTIwJTIwJTIwdGVtcGVyYXR1cmUlM0ElMjAwLjUlMEElMjAlMjAlMjAlMjBtYXhfbmV3X3Rva2VucyUzQSUyMDI1NiUyMCUyMyUyMFRoaXMlMjB3aWxsJTIwb3ZlcmlkZSUyMHRoZSUyMGRlZmF1bHQlMjBmcm9tJTIwdGhlJTIwdGFza3MlMjBjb25maWclMEElMjAlMjAlMjAlMjB0b3BfcCUzQSUyMDAuOSUwQSUyMCUyMCUyMCUyMHNlZWQlM0ElMjAwJTBBJTIwJTIwJTIwJTIwcmVwZXRpdGlvbl9wZW5hbHR5JTNBJTIwMS4wJTBBJTIwJTIwJTIwJTIwZnJlcXVlbmN5X3BlbmFsdHklM0ElMjAwLjA=",highlighted:`<span class="hljs-attr">model:</span>
  <span class="hljs-attr">base_params:</span>
    <span class="hljs-attr">model_name:</span> <span class="hljs-string">&quot;openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B&quot;</span>
    <span class="hljs-attr">base_url:</span> <span class="hljs-string">&quot;https://router.huggingface.co/hf-inference/v1&quot;</span>
    <span class="hljs-attr">api_key:</span> <span class="hljs-string">&quot;YOUR KEY&quot;</span> <span class="hljs-comment"># remove or keep empty as needed</span>
  <span class="hljs-attr">generation:</span>
    <span class="hljs-attr">temperature:</span> <span class="hljs-number">0.5</span>
    <span class="hljs-attr">max_new_tokens:</span> <span class="hljs-number">256</span> <span class="hljs-comment"># This will overide the default from the tasks config</span>
    <span class="hljs-attr">top_p:</span> <span class="hljs-number">0.9</span>
    <span class="hljs-attr">seed:</span> <span class="hljs-number">0</span>
    <span class="hljs-attr">repetition_penalty:</span> <span class="hljs-number">1.0</span>
    <span class="hljs-attr">frequency_penalty:</span> <span class="hljs-number">0.0</span>`,wrap:!1}}),S=new ce({props:{code:"bGlnaHRldmFsJTIwZW5kcG9pbnQlMjBsaXRlbGxtJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIyZXhhbXBsZXMlMkZtb2RlbF9jb25maWdzJTJGbGl0ZWxsbV9tb2RlbC55YW1sJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybGlnaHRldmFsJTdDZ3NtOGslN0MwJTdDMCUyMg==",highlighted:`lighteval endpoint litellm \\
    <span class="hljs-string">&quot;examples/model_configs/litellm_model.yaml&quot;</span> \\
    <span class="hljs-string">&quot;lighteval|gsm8k|0|0&quot;</span>`,wrap:!1}}),W=new Ve({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/use-litellm-as-backend.mdx"}}),{c(){p=i("meta"),T=a(),M=i("p"),X=a(),r(J.$$.fragment),Q=a(),g=i("p"),g.textContent=he,L=a(),d=i("p"),d.innerHTML=fe,N=a(),r(y.$$.fragment),q=a(),r(U.$$.fragment),P=a(),r(b.$$.fragment),D=a(),I=i("p"),I.textContent=we,F=a(),$=i("p"),$.textContent=je,z=a(),r(v.$$.fragment),Y=a(),r(C.$$.fragment),O=a(),B=i("p"),B.textContent=Te,K=a(),k=i("p"),k.innerHTML=Je,ee=a(),_=i("p"),_.textContent=ge,te=a(),A=i("p"),A.innerHTML=de,le=a(),Z=i("p"),Z.textContent=ye,se=a(),x=i("p"),x.innerHTML=Ue,ae=a(),E=i("p"),E.textContent=be,ne=a(),r(j.$$.fragment),pe=a(),r(G.$$.fragment),ie=a(),V=i("p"),V.textContent=Ie,me=a(),r(S.$$.fragment),oe=a(),r(W.$$.fragment),Me=a(),R=i("p"),this.h()},l(e){const t=xe("svelte-u9bgzb",document.head);p=m(t,"META",{name:!0,content:!0}),t.forEach(l),T=n(e),M=m(e,"P",{}),ve(M).forEach(l),X=n(e),c(J.$$.fragment,e),Q=n(e),g=m(e,"P",{"data-svelte-h":!0}),o(g)!=="svelte-1khirk"&&(g.textContent=he),L=n(e),d=m(e,"P",{"data-svelte-h":!0}),o(d)!=="svelte-17swpu2"&&(d.innerHTML=fe),N=n(e),c(y.$$.fragment,e),q=n(e),c(U.$$.fragment,e),P=n(e),c(b.$$.fragment,e),D=n(e),I=m(e,"P",{"data-svelte-h":!0}),o(I)!=="svelte-1yct201"&&(I.textContent=we),F=n(e),$=m(e,"P",{"data-svelte-h":!0}),o($)!=="svelte-15ud26h"&&($.textContent=je),z=n(e),c(v.$$.fragment,e),Y=n(e),c(C.$$.fragment,e),O=n(e),B=m(e,"P",{"data-svelte-h":!0}),o(B)!=="svelte-4otdia"&&(B.textContent=Te),K=n(e),k=m(e,"P",{"data-svelte-h":!0}),o(k)!=="svelte-1p1c49"&&(k.innerHTML=Je),ee=n(e),_=m(e,"P",{"data-svelte-h":!0}),o(_)!=="svelte-i6oxxx"&&(_.textContent=ge),te=n(e),A=m(e,"P",{"data-svelte-h":!0}),o(A)!=="svelte-e088um"&&(A.innerHTML=de),le=n(e),Z=m(e,"P",{"data-svelte-h":!0}),o(Z)!=="svelte-tpzrbu"&&(Z.textContent=ye),se=n(e),x=m(e,"P",{"data-svelte-h":!0}),o(x)!=="svelte-xhodo0"&&(x.innerHTML=Ue),ae=n(e),E=m(e,"P",{"data-svelte-h":!0}),o(E)!=="svelte-oho0xe"&&(E.textContent=be),ne=n(e),c(j.$$.fragment,e),pe=n(e),c(G.$$.fragment,e),ie=n(e),V=m(e,"P",{"data-svelte-h":!0}),o(V)!=="svelte-1alvfsp"&&(V.textContent=Ie),me=n(e),c(S.$$.fragment,e),oe=n(e),c(W.$$.fragment,e),Me=n(e),R=m(e,"P",{}),ve(R).forEach(l),this.h()},h(){Ce(p,"name","hf:doc:metadata"),Ce(p,"content",Xe)},m(e,t){Ee(document.head,p),s(e,T,t),s(e,M,t),s(e,X,t),u(J,e,t),s(e,Q,t),s(e,g,t),s(e,L,t),s(e,d,t),s(e,N,t),u(y,e,t),s(e,q,t),u(U,e,t),s(e,P,t),u(b,e,t),s(e,D,t),s(e,I,t),s(e,F,t),s(e,$,t),s(e,z,t),u(v,e,t),s(e,Y,t),u(C,e,t),s(e,O,t),s(e,B,t),s(e,K,t),s(e,k,t),s(e,ee,t),s(e,_,t),s(e,te,t),s(e,A,t),s(e,le,t),s(e,Z,t),s(e,se,t),s(e,x,t),s(e,ae,t),s(e,E,t),s(e,ne,t),u(j,e,t),s(e,pe,t),u(G,e,t),s(e,ie,t),s(e,V,t),s(e,me,t),u(S,e,t),s(e,oe,t),u(W,e,t),s(e,Me,t),s(e,R,t),re=!0},p(e,[t]){const $e={};t&2&&($e.$$scope={dirty:t,ctx:e}),j.$set($e)},i(e){re||(h(J.$$.fragment,e),h(y.$$.fragment,e),h(U.$$.fragment,e),h(b.$$.fragment,e),h(v.$$.fragment,e),h(C.$$.fragment,e),h(j.$$.fragment,e),h(G.$$.fragment,e),h(S.$$.fragment,e),h(W.$$.fragment,e),re=!0)},o(e){f(J.$$.fragment,e),f(y.$$.fragment,e),f(U.$$.fragment,e),f(b.$$.fragment,e),f(v.$$.fragment,e),f(C.$$.fragment,e),f(j.$$.fragment,e),f(G.$$.fragment,e),f(S.$$.fragment,e),f(W.$$.fragment,e),re=!1},d(e){e&&(l(T),l(M),l(X),l(Q),l(g),l(L),l(d),l(N),l(q),l(P),l(D),l(I),l(F),l($),l(z),l(Y),l(O),l(B),l(K),l(k),l(ee),l(_),l(te),l(A),l(le),l(Z),l(se),l(x),l(ae),l(E),l(ne),l(pe),l(ie),l(V),l(me),l(oe),l(Me),l(R)),l(p),w(J,e),w(y,e),w(U,e),w(b,e),w(v,e),w(C,e),w(j,e),w(G,e),w(S,e),w(W,e)}}}const Xe='{"title":"Litellm as backend","local":"litellm-as-backend","sections":[{"title":"Quick use","local":"quick-use","sections":[],"depth":2},{"title":"Using a config file","local":"using-a-config-file","sections":[],"depth":2},{"title":"Use Hugging Face Inference Providers","local":"use-hugging-face-inference-providers","sections":[],"depth":2}],"depth":1}';function Re(H){return ke(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pe extends Ae{constructor(p){super(),Ze(this,p,Re,We,Be,{})}}export{Pe as component};