lighteval/pr_1027/en/_app/immutable/nodes/23.eed9f4e5.js

import{s as Se,n as Ze,o as Ae}from"../chunks/scheduler.5f3e6389.js";import{S as Be,i as Ve,e as i,s as a,c as p,h as He,a as r,d as l,b as n,f as xe,g as o,j as $,k as Ue,l as Ee,m as s,n as m,t as M,o as u,p as f}from"../chunks/index.373ab25c.js";import{C as qe,H as U,E as We}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.c2e0d06d.js";import{C as de}from"../chunks/CodeBlock.cd35d790.js";function Fe(we){let c,R,F,Q,d,X,w,P,T,Te=`Lighteval allows you to use LiteLLM as a backend, enabling you to call all LLM APIs
using the OpenAI format. LiteLLM supports various providers including Bedrock, Hugging Face, Vertex AI, Together AI, Azure,
OpenAI, Groq, and many others.`,N,g,he='<p>Documentation for available APIs and compatible endpoints can be found <a href="https://docs.litellm.ai/docs/" rel="nofollow">here</a>.</p>',z,h,O,C,Y,L,D,j,Ce=`LiteLLM allows generation with any OpenAI-compatible endpoint. For example, you
can evaluate a model running on a local VLLM server.`,K,b,Le="To do so, you will need to use a configuration file like this:",ee,v,te,I,le,J,je="LiteLLM supports a wide range of LLM providers:",se,_,ae,k,be='all cloud providers can be found in the <a href="https://docs.litellm.ai/docs/providers" rel="nofollow">litellm documentation</a>.',ne,x,ie,S,ve="<li><strong>VLLM</strong>: Local VLLM servers</li> <li><strong>Hugging Face</strong>: Local Hugging Face models</li> <li><strong>Custom endpoints</strong>: Any OpenAI-compatible API</li>",pe,Z,re,A,oe,B,Ie="To use with a local VLLM server:",me,V,Je="<li>Start your VLLM server:</li>",Me,H,ue,y,_e="<li>Configure LiteLLM to use the local server:</li>",fe,E,$e,q,ke='For more detailed error handling and debugging, refer to the <a href="https://docs.litellm.ai/docs/" rel="nofollow">LiteLLM documentation</a>.',ce,W,ge,G,ye;return d=new qe({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new U({props:{title:"Using LiteLLM as Backend",local:"using-litellm-as-backend",headingTag:"h1"}}),h=new U({props:{title:"Basic Usage",local:"basic-usage",headingTag:"h2"}}),C=new de({props:{code:"bGlnaHRldmFsJTIwZW5kcG9pbnQlMjBsaXRlbGxtJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIycHJvdmlkZXIlM0RvcGVuYWklMkNtb2RlbF9uYW1lJTNEZ3B0LTMuNS10dXJibyUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMmxpZ2h0ZXZhbCU3Q2dzbThrJTdDMCUyMg==",highlighted:`lighteval endpoint litellm \\
    <span class="hljs-string">&quot;provider=openai,model_name=gpt-3.5-turbo&quot;</span> \\
    <span class="hljs-string">&quot;lighteval|gsm8k|0&quot;</span>`,wrap:!1}}),L=new U({props:{title:"Using a Configuration File",local:"using-a-configuration-file",headingTag:"h2"}}),v=new de({props:{code:"bW9kZWxfcGFyYW1ldGVycyUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0ElMjAlMjJvcGVuYWklMkZkZWVwc2Vlay1haSUyRkRlZXBTZWVrLVIxLURpc3RpbGwtUXdlbi0zMkIlMjIlMEElMjAlMjAlMjAlMjBiYXNlX3VybCUzQSUyMCUyMlVSTF9PRl9USEVfRU5EUE9JTlRfWU9VX1dBTlRfVE9fVVNFJTIyJTBBJTIwJTIwJTIwJTIwYXBpX2tleSUzQSUyMCUyMiUyMiUyMCUyMyUyMFJlbW92ZSUyMG9yJTIwa2VlcCUyMGVtcHR5JTIwYXMlMjBuZWVkZWQlMEElMjAlMjAlMjAlMjBnZW5lcmF0aW9uX3BhcmFtZXRlcnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzQSUyMDAuNSUwQSUyMCUyMCUyMCUyMCUyMCUyMG1heF9uZXdfdG9rZW5zJTNBJTIwMjU2JTBBJTIwJTIwJTIwJTIwJTIwJTIwc3RvcF90b2tlbnMlM0ElMjAlNUIlMjIlMjIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjB0b3BfcCUzQSUyMDAuOSUwQSUyMCUyMCUyMCUyMCUyMCUyMHNlZWQlM0ElMjAwJTBBJTIwJTIwJTIwJTIwJTIwJTIwcmVwZXRpdGlvbl9wZW5hbHR5JTNBJTIwMS4wJTBBJTIwJTIwJTIwJTIwJTIwJTIwZnJlcXVlbmN5X3BlbmFsdHklM0ElMjAwLjA=",highlighted:`<span class="hljs-attr">model_parameters:</span>
    <span class="hljs-attr">model_name:</span> <span class="hljs-string">&quot;openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B&quot;</span>
    <span class="hljs-attr">base_url:</span> <span class="hljs-string">&quot;URL_OF_THE_ENDPOINT_YOU_WANT_TO_USE&quot;</span>
    <span class="hljs-attr">api_key:</span> <span class="hljs-string">&quot;&quot;</span> <span class="hljs-comment"># Remove or keep empty as needed</span>
    <span class="hljs-attr">generation_parameters:</span>
      <span class="hljs-attr">temperature:</span> <span class="hljs-number">0.5</span>
      <span class="hljs-attr">max_new_tokens:</span> <span class="hljs-number">256</span>
      <span class="hljs-attr">stop_tokens:</span> [<span class="hljs-string">&quot;&quot;</span>]
      <span class="hljs-attr">top_p:</span> <span class="hljs-number">0.9</span>
      <span class="hljs-attr">seed:</span> <span class="hljs-number">0</span>
      <span class="hljs-attr">repetition_penalty:</span> <span class="hljs-number">1.0</span>
      <span class="hljs-attr">frequency_penalty:</span> <span class="hljs-number">0.0</span>`,wrap:!1}}),I=new U({props:{title:"Supported Providers",local:"supported-providers",headingTag:"h2"}}),_=new U({props:{title:"Cloud Providers",local:"cloud-providers",headingTag:"h3"}}),x=new U({props:{title:"Local/On-Premise",local:"localon-premise",headingTag:"h3"}}),Z=new U({props:{title:"Using with Local Models",local:"using-with-local-models",headingTag:"h2"}}),A=new U({props:{title:"VLLM Server",local:"vllm-server",headingTag:"h3"}}),H=new de({props:{code:"dmxsbSUyMHNlcnZlJTIwSHVnZ2luZ0ZhY2VINCUyRnplcGh5ci03Yi1iZXRhJTIwLS1ob3N0JTIwMC4wLjAuMCUyMC0tcG9ydCUyMDgwMDA=",highlighted:"vllm serve HuggingFaceH4/zephyr-7b-beta --host 0.0.0.0 --port 8000",wrap:!1}}),E=new de({props:{code:"bW9kZWxfcGFyYW1ldGVycyUzQSUwQSUyMCUyMCUyMCUyMHByb3ZpZGVyJTNBJTIwJTIyb3BlbmFpJTIyJTBBJTIwJTIwJTIwJTIwbW9kZWxfbmFtZSUzQSUyMCUyMkh1Z2dpbmdGYWNlSDQlMkZ6ZXBoeXItN2ItYmV0YSUyMiUwQSUyMCUyMCUyMCUyMGJhc2VfdXJsJTNBJTIwJTIyaHR0cCUzQSUyRiUyRmxvY2FsaG9zdCUzQTgwMDAlMkZ2MSUyMiUwQSUyMCUyMCUyMCUyMGFwaV9rZXklM0ElMjAlMjIlMjI=",highlighted:`<span class="hljs-attr">model_parameters:</span>
    <span class="hljs-attr">provider:</span> <span class="hljs-string">&quot;openai&quot;</span>
    <span class="hljs-attr">model_name:</span> <span class="hljs-string">&quot;HuggingFaceH4/zephyr-7b-beta&quot;</span>
    <span class="hljs-attr">base_url:</span> <span class="hljs-string">&quot;http://localhost:8000/v1&quot;</span>
    <span class="hljs-attr">api_key:</span> <span class="hljs-string">&quot;&quot;</span>`,wrap:!1}}),W=new We({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/use-litellm-as-backend.mdx"}}),{c(){c=i("meta"),R=a(),F=i("p"),Q=a(),p(d.$$.fragment),X=a(),p(w.$$.fragment),P=a(),T=i("p"),T.textContent=Te,N=a(),g=i("blockquote"),g.innerHTML=he,z=a(),p(h.$$.fragment),O=a(),p(C.$$.fragment),Y=a(),p(L.$$.fragment),D=a(),j=i("p"),j.textContent=Ce,K=a(),b=i("p"),b.textContent=Le,ee=a(),p(v.$$.fragment),te=a(),p(I.$$.fragment),le=a(),J=i("p"),J.textContent=je,se=a(),p(_.$$.fragment),ae=a(),k=i("p"),k.innerHTML=be,ne=a(),p(x.$$.fragment),ie=a(),S=i("ul"),S.innerHTML=ve,pe=a(),p(Z.$$.fragment),re=a(),p(A.$$.fragment),oe=a(),B=i("p"),B.textContent=Ie,me=a(),V=i("ol"),V.innerHTML=Je,Me=a(),p(H.$$.fragment),ue=a(),y=i("ol"),y.innerHTML=_e,fe=a(),p(E.$$.fragment),$e=a(),q=i("p"),q.innerHTML=ke,ce=a(),p(W.$$.fragment),ge=a(),G=i("p"),this.h()},l(e){const t=He("svelte-u9bgzb",document.head);c=r(t,"META",{name:!0,content:!0}),t.forEach(l),R=n(e),F=r(e,"P",{}),xe(F).forEach(l),Q=n(e),o(d.$$.fragment,e),X=n(e),o(w.$$.fragment,e),P=n(e),T=r(e,"P",{"data-svelte-h":!0}),$(T)!=="svelte-a1o746"&&(T.textContent=Te),N=n(e),g=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),$(g)!=="svelte-tnom0v"&&(g.innerHTML=he),z=n(e),o(h.$$.fragment,e),O=n(e),o(C.$$.fragment,e),Y=n(e),o(L.$$.fragment,e),D=n(e),j=r(e,"P",{"data-svelte-h":!0}),$(j)!=="svelte-3l1m4a"&&(j.textContent=Ce),K=n(e),b=r(e,"P",{"data-svelte-h":!0}),$(b)!=="svelte-pxnxuz"&&(b.textContent=Le),ee=n(e),o(v.$$.fragment,e),te=n(e),o(I.$$.fragment,e),le=n(e),J=r(e,"P",{"data-svelte-h":!0}),$(J)!=="svelte-10eon5u"&&(J.textContent=je),se=n(e),o(_.$$.fragment,e),ae=n(e),k=r(e,"P",{"data-svelte-h":!0}),$(k)!=="svelte-9dfpjp"&&(k.innerHTML=be),ne=n(e),o(x.$$.fragment,e),ie=n(e),S=r(e,"UL",{"data-svelte-h":!0}),$(S)!=="svelte-1craf7w"&&(S.innerHTML=ve),pe=n(e),o(Z.$$.fragment,e),re=n(e),o(A.$$.fragment,e),oe=n(e),B=r(e,"P",{"data-svelte-h":!0}),$(B)!=="svelte-tacy60"&&(B.textContent=Ie),me=n(e),V=r(e,"OL",{"data-svelte-h":!0}),$(V)!=="svelte-xokqka"&&(V.innerHTML=Je),Me=n(e),o(H.$$.fragment,e),ue=n(e),y=r(e,"OL",{start:!0,"data-svelte-h":!0}),$(y)!=="svelte-1ujc1uy"&&(y.innerHTML=_e),fe=n(e),o(E.$$.fragment,e),$e=n(e),q=r(e,"P",{"data-svelte-h":!0}),$(q)!=="svelte-fyarqi"&&(q.innerHTML=ke),ce=n(e),o(W.$$.fragment,e),ge=n(e),G=r(e,"P",{}),xe(G).forEach(l),this.h()},h(){Ue(c,"name","hf:doc:metadata"),Ue(c,"content",Ge),Ue(g,"class","tip"),Ue(y,"start","2")},m(e,t){Ee(document.head,c),s(e,R,t),s(e,F,t),s(e,Q,t),m(d,e,t),s(e,X,t),m(w,e,t),s(e,P,t),s(e,T,t),s(e,N,t),s(e,g,t),s(e,z,t),m(h,e,t),s(e,O,t),m(C,e,t),s(e,Y,t),m(L,e,t),s(e,D,t),s(e,j,t),s(e,K,t),s(e,b,t),s(e,ee,t),m(v,e,t),s(e,te,t),m(I,e,t),s(e,le,t),s(e,J,t),s(e,se,t),m(_,e,t),s(e,ae,t),s(e,k,t),s(e,ne,t),m(x,e,t),s(e,ie,t),s(e,S,t),s(e,pe,t),m(Z,e,t),s(e,re,t),m(A,e,t),s(e,oe,t),s(e,B,t),s(e,me,t),s(e,V,t),s(e,Me,t),m(H,e,t),s(e,ue,t),s(e,y,t),s(e,fe,t),m(E,e,t),s(e,$e,t),s(e,q,t),s(e,ce,t),m(W,e,t),s(e,ge,t),s(e,G,t),ye=!0},p:Ze,i(e){ye||(M(d.$$.fragment,e),M(w.$$.fragment,e),M(h.$$.fragment,e),M(C.$$.fragment,e),M(L.$$.fragment,e),M(v.$$.fragment,e),M(I.$$.fragment,e),M(_.$$.fragment,e),M(x.$$.fragment,e),M(Z.$$.fragment,e),M(A.$$.fragment,e),M(H.$$.fragment,e),M(E.$$.fragment,e),M(W.$$.fragment,e),ye=!0)},o(e){u(d.$$.fragment,e),u(w.$$.fragment,e),u(h.$$.fragment,e),u(C.$$.fragment,e),u(L.$$.fragment,e),u(v.$$.fragment,e),u(I.$$.fragment,e),u(_.$$.fragment,e),u(x.$$.fragment,e),u(Z.$$.fragment,e),u(A.$$.fragment,e),u(H.$$.fragment,e),u(E.$$.fragment,e),u(W.$$.fragment,e),ye=!1},d(e){e&&(l(R),l(F),l(Q),l(X),l(P),l(T),l(N),l(g),l(z),l(O),l(Y),l(D),l(j),l(K),l(b),l(ee),l(te),l(le),l(J),l(se),l(ae),l(k),l(ne),l(ie),l(S),l(pe),l(re),l(oe),l(B),l(me),l(V),l(Me),l(ue),l(y),l(fe),l($e),l(q),l(ce),l(ge),l(G)),l(c),f(d,e),f(w,e),f(h,e),f(C,e),f(L,e),f(v,e),f(I,e),f(_,e),f(x,e),f(Z,e),f(A,e),f(H,e),f(E,e),f(W,e)}}}const Ge='{"title":"Using LiteLLM as Backend","local":"using-litellm-as-backend","sections":[{"title":"Basic Usage","local":"basic-usage","sections":[],"depth":2},{"title":"Using a Configuration File","local":"using-a-configuration-file","sections":[],"depth":2},{"title":"Supported Providers","local":"supported-providers","sections":[{"title":"Cloud Providers","local":"cloud-providers","sections":[],"depth":3},{"title":"Local/On-Premise","local":"localon-premise","sections":[],"depth":3}],"depth":2},{"title":"Using with Local Models","local":"using-with-local-models","sections":[{"title":"VLLM Server","local":"vllm-server","sections":[],"depth":3}],"depth":2}],"depth":1}';function Re(we){return Ae(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ze extends Be{constructor(c){super(),Ve(this,c,Re,Fe,Se,{})}}export{ze as component};