lighteval/pr_985/en/_app/immutable/nodes/23.b827da4e.js

import{s as ke,n as Se,o as Ae}from"../chunks/scheduler.7da89386.js";import{S as Ze,i as xe,g as i,s as a,r,A as Be,h as p,f as l,c as n,j as _e,u as o,x as c,k as $e,y as Ve,a as s,v as m,d as M,t as u,w as f}from"../chunks/index.20910acc.js";import{C as ye}from"../chunks/CodeBlock.143bd81e.js";import{H as d,E as He}from"../chunks/getInferenceSnippets.375cdad5.js";function Ee(de){let g,G,W,R,U,Q,w,Ue=`Lighteval allows you to use LiteLLM as a backend, enabling you to call all LLM APIs
using the OpenAI format. LiteLLM supports various providers including Bedrock, Hugging Face, Vertex AI, Together AI, Azure,
OpenAI, Groq, and many others.`,X,$,we='<p>Documentation for available APIs and compatible endpoints can be found <a href="https://docs.litellm.ai/docs/" rel="nofollow">here</a>.</p>',P,h,N,T,z,C,O,j,he=`LiteLLM allows generation with any OpenAI-compatible endpoint. For example, you
can evaluate a model running on a local VLLM server.`,Y,L,Te="To do so, you will need to use a configuration file like this:",D,b,K,v,ee,I,Ce="LiteLLM supports a wide range of LLM providers:",te,J,le,_,je='all cloud providers can be found in the <a href="https://docs.litellm.ai/docs/providers" rel="nofollow">litellm documentation</a>.',se,k,ae,S,Le="<li><strong>VLLM</strong>: Local VLLM servers</li> <li><strong>Hugging Face</strong>: Local Hugging Face models</li> <li><strong>Custom endpoints</strong>: Any OpenAI-compatible API</li>",ne,A,ie,Z,pe,x,be="To use with a local VLLM server:",re,B,ve="<li>Start your VLLM server:</li>",oe,V,me,y,Ie="<li>Configure LiteLLM to use the local server:</li>",Me,H,ue,E,Je='For more detailed error handling and debugging, refer to the <a href="https://docs.litellm.ai/docs/" rel="nofollow">LiteLLM documentation</a>.',fe,q,ce,F,ge;return U=new d({props:{title:"Using LiteLLM as Backend",local:"using-litellm-as-backend",headingTag:"h1"}}),h=new d({props:{title:"Basic Usage",local:"basic-usage",headingTag:"h2"}}),T=new ye({props:{code:"bGlnaHRldmFsJTIwZW5kcG9pbnQlMjBsaXRlbGxtJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIycHJvdmlkZXIlM0RvcGVuYWklMkNtb2RlbF9uYW1lJTNEZ3B0LTMuNS10dXJibyUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMmxpZ2h0ZXZhbCU3Q2dzbThrJTdDMCUyMg==",highlighted:`lighteval endpoint litellm \\
    <span class="hljs-string">&quot;provider=openai,model_name=gpt-3.5-turbo&quot;</span> \\
    <span class="hljs-string">&quot;lighteval|gsm8k|0&quot;</span>`,wrap:!1}}),C=new d({props:{title:"Using a Configuration File",local:"using-a-configuration-file",headingTag:"h2"}}),b=new ye({props:{code:"bW9kZWxfcGFyYW1ldGVycyUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0ElMjAlMjJvcGVuYWklMkZkZWVwc2Vlay1haSUyRkRlZXBTZWVrLVIxLURpc3RpbGwtUXdlbi0zMkIlMjIlMEElMjAlMjAlMjAlMjBiYXNlX3VybCUzQSUyMCUyMlVSTF9PRl9USEVfRU5EUE9JTlRfWU9VX1dBTlRfVE9fVVNFJTIyJTBBJTIwJTIwJTIwJTIwYXBpX2tleSUzQSUyMCUyMiUyMiUyMCUyMyUyMFJlbW92ZSUyMG9yJTIwa2VlcCUyMGVtcHR5JTIwYXMlMjBuZWVkZWQlMEElMjAlMjAlMjAlMjBnZW5lcmF0aW9uX3BhcmFtZXRlcnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzQSUyMDAuNSUwQSUyMCUyMCUyMCUyMCUyMCUyMG1heF9uZXdfdG9rZW5zJTNBJTIwMjU2JTBBJTIwJTIwJTIwJTIwJTIwJTIwc3RvcF90b2tlbnMlM0ElMjAlNUIlMjIlMjIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjB0b3BfcCUzQSUyMDAuOSUwQSUyMCUyMCUyMCUyMCUyMCUyMHNlZWQlM0ElMjAwJTBBJTIwJTIwJTIwJTIwJTIwJTIwcmVwZXRpdGlvbl9wZW5hbHR5JTNBJTIwMS4wJTBBJTIwJTIwJTIwJTIwJTIwJTIwZnJlcXVlbmN5X3BlbmFsdHklM0ElMjAwLjA=",highlighted:`<span class="hljs-attr">model_parameters:</span>
    <span class="hljs-attr">model_name:</span> <span class="hljs-string">&quot;openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B&quot;</span>
    <span class="hljs-attr">base_url:</span> <span class="hljs-string">&quot;URL_OF_THE_ENDPOINT_YOU_WANT_TO_USE&quot;</span>
    <span class="hljs-attr">api_key:</span> <span class="hljs-string">&quot;&quot;</span> <span class="hljs-comment"># Remove or keep empty as needed</span>
    <span class="hljs-attr">generation_parameters:</span>
      <span class="hljs-attr">temperature:</span> <span class="hljs-number">0.5</span>
      <span class="hljs-attr">max_new_tokens:</span> <span class="hljs-number">256</span>
      <span class="hljs-attr">stop_tokens:</span> [<span class="hljs-string">&quot;&quot;</span>]
      <span class="hljs-attr">top_p:</span> <span class="hljs-number">0.9</span>
      <span class="hljs-attr">seed:</span> <span class="hljs-number">0</span>
      <span class="hljs-attr">repetition_penalty:</span> <span class="hljs-number">1.0</span>
      <span class="hljs-attr">frequency_penalty:</span> <span class="hljs-number">0.0</span>`,wrap:!1}}),v=new d({props:{title:"Supported Providers",local:"supported-providers",headingTag:"h2"}}),J=new d({props:{title:"Cloud Providers",local:"cloud-providers",headingTag:"h3"}}),k=new d({props:{title:"Local/On-Premise",local:"localon-premise",headingTag:"h3"}}),A=new d({props:{title:"Using with Local Models",local:"using-with-local-models",headingTag:"h2"}}),Z=new d({props:{title:"VLLM Server",local:"vllm-server",headingTag:"h3"}}),V=new ye({props:{code:"dmxsbSUyMHNlcnZlJTIwSHVnZ2luZ0ZhY2VINCUyRnplcGh5ci03Yi1iZXRhJTIwLS1ob3N0JTIwMC4wLjAuMCUyMC0tcG9ydCUyMDgwMDA=",highlighted:"vllm serve HuggingFaceH4/zephyr-7b-beta --host 0.0.0.0 --port 8000",wrap:!1}}),H=new ye({props:{code:"bW9kZWxfcGFyYW1ldGVycyUzQSUwQSUyMCUyMCUyMCUyMHByb3ZpZGVyJTNBJTIwJTIyb3BlbmFpJTIyJTBBJTIwJTIwJTIwJTIwbW9kZWxfbmFtZSUzQSUyMCUyMkh1Z2dpbmdGYWNlSDQlMkZ6ZXBoeXItN2ItYmV0YSUyMiUwQSUyMCUyMCUyMCUyMGJhc2VfdXJsJTNBJTIwJTIyaHR0cCUzQSUyRiUyRmxvY2FsaG9zdCUzQTgwMDAlMkZ2MSUyMiUwQSUyMCUyMCUyMCUyMGFwaV9rZXklM0ElMjAlMjIlMjI=",highlighted:`<span class="hljs-attr">model_parameters:</span>
    <span class="hljs-attr">provider:</span> <span class="hljs-string">&quot;openai&quot;</span>
    <span class="hljs-attr">model_name:</span> <span class="hljs-string">&quot;HuggingFaceH4/zephyr-7b-beta&quot;</span>
    <span class="hljs-attr">base_url:</span> <span class="hljs-string">&quot;http://localhost:8000/v1&quot;</span>
    <span class="hljs-attr">api_key:</span> <span class="hljs-string">&quot;&quot;</span>`,wrap:!1}}),q=new He({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/use-litellm-as-backend.mdx"}}),{c(){g=i("meta"),G=a(),W=i("p"),R=a(),r(U.$$.fragment),Q=a(),w=i("p"),w.textContent=Ue,X=a(),$=i("blockquote"),$.innerHTML=we,P=a(),r(h.$$.fragment),N=a(),r(T.$$.fragment),z=a(),r(C.$$.fragment),O=a(),j=i("p"),j.textContent=he,Y=a(),L=i("p"),L.textContent=Te,D=a(),r(b.$$.fragment),K=a(),r(v.$$.fragment),ee=a(),I=i("p"),I.textContent=Ce,te=a(),r(J.$$.fragment),le=a(),_=i("p"),_.innerHTML=je,se=a(),r(k.$$.fragment),ae=a(),S=i("ul"),S.innerHTML=Le,ne=a(),r(A.$$.fragment),ie=a(),r(Z.$$.fragment),pe=a(),x=i("p"),x.textContent=be,re=a(),B=i("ol"),B.innerHTML=ve,oe=a(),r(V.$$.fragment),me=a(),y=i("ol"),y.innerHTML=Ie,Me=a(),r(H.$$.fragment),ue=a(),E=i("p"),E.innerHTML=Je,fe=a(),r(q.$$.fragment),ce=a(),F=i("p"),this.h()},l(e){const t=Be("svelte-u9bgzb",document.head);g=p(t,"META",{name:!0,content:!0}),t.forEach(l),G=n(e),W=p(e,"P",{}),_e(W).forEach(l),R=n(e),o(U.$$.fragment,e),Q=n(e),w=p(e,"P",{"data-svelte-h":!0}),c(w)!=="svelte-a1o746"&&(w.textContent=Ue),X=n(e),$=p(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),c($)!=="svelte-tnom0v"&&($.innerHTML=we),P=n(e),o(h.$$.fragment,e),N=n(e),o(T.$$.fragment,e),z=n(e),o(C.$$.fragment,e),O=n(e),j=p(e,"P",{"data-svelte-h":!0}),c(j)!=="svelte-3l1m4a"&&(j.textContent=he),Y=n(e),L=p(e,"P",{"data-svelte-h":!0}),c(L)!=="svelte-pxnxuz"&&(L.textContent=Te),D=n(e),o(b.$$.fragment,e),K=n(e),o(v.$$.fragment,e),ee=n(e),I=p(e,"P",{"data-svelte-h":!0}),c(I)!=="svelte-10eon5u"&&(I.textContent=Ce),te=n(e),o(J.$$.fragment,e),le=n(e),_=p(e,"P",{"data-svelte-h":!0}),c(_)!=="svelte-9dfpjp"&&(_.innerHTML=je),se=n(e),o(k.$$.fragment,e),ae=n(e),S=p(e,"UL",{"data-svelte-h":!0}),c(S)!=="svelte-1craf7w"&&(S.innerHTML=Le),ne=n(e),o(A.$$.fragment,e),ie=n(e),o(Z.$$.fragment,e),pe=n(e),x=p(e,"P",{"data-svelte-h":!0}),c(x)!=="svelte-tacy60"&&(x.textContent=be),re=n(e),B=p(e,"OL",{"data-svelte-h":!0}),c(B)!=="svelte-xokqka"&&(B.innerHTML=ve),oe=n(e),o(V.$$.fragment,e),me=n(e),y=p(e,"OL",{start:!0,"data-svelte-h":!0}),c(y)!=="svelte-1ujc1uy"&&(y.innerHTML=Ie),Me=n(e),o(H.$$.fragment,e),ue=n(e),E=p(e,"P",{"data-svelte-h":!0}),c(E)!=="svelte-fyarqi"&&(E.innerHTML=Je),fe=n(e),o(q.$$.fragment,e),ce=n(e),F=p(e,"P",{}),_e(F).forEach(l),this.h()},h(){$e(g,"name","hf:doc:metadata"),$e(g,"content",qe),$e($,"class","tip"),$e(y,"start","2")},m(e,t){Ve(document.head,g),s(e,G,t),s(e,W,t),s(e,R,t),m(U,e,t),s(e,Q,t),s(e,w,t),s(e,X,t),s(e,$,t),s(e,P,t),m(h,e,t),s(e,N,t),m(T,e,t),s(e,z,t),m(C,e,t),s(e,O,t),s(e,j,t),s(e,Y,t),s(e,L,t),s(e,D,t),m(b,e,t),s(e,K,t),m(v,e,t),s(e,ee,t),s(e,I,t),s(e,te,t),m(J,e,t),s(e,le,t),s(e,_,t),s(e,se,t),m(k,e,t),s(e,ae,t),s(e,S,t),s(e,ne,t),m(A,e,t),s(e,ie,t),m(Z,e,t),s(e,pe,t),s(e,x,t),s(e,re,t),s(e,B,t),s(e,oe,t),m(V,e,t),s(e,me,t),s(e,y,t),s(e,Me,t),m(H,e,t),s(e,ue,t),s(e,E,t),s(e,fe,t),m(q,e,t),s(e,ce,t),s(e,F,t),ge=!0},p:Se,i(e){ge||(M(U.$$.fragment,e),M(h.$$.fragment,e),M(T.$$.fragment,e),M(C.$$.fragment,e),M(b.$$.fragment,e),M(v.$$.fragment,e),M(J.$$.fragment,e),M(k.$$.fragment,e),M(A.$$.fragment,e),M(Z.$$.fragment,e),M(V.$$.fragment,e),M(H.$$.fragment,e),M(q.$$.fragment,e),ge=!0)},o(e){u(U.$$.fragment,e),u(h.$$.fragment,e),u(T.$$.fragment,e),u(C.$$.fragment,e),u(b.$$.fragment,e),u(v.$$.fragment,e),u(J.$$.fragment,e),u(k.$$.fragment,e),u(A.$$.fragment,e),u(Z.$$.fragment,e),u(V.$$.fragment,e),u(H.$$.fragment,e),u(q.$$.fragment,e),ge=!1},d(e){e&&(l(G),l(W),l(R),l(Q),l(w),l(X),l($),l(P),l(N),l(z),l(O),l(j),l(Y),l(L),l(D),l(K),l(ee),l(I),l(te),l(le),l(_),l(se),l(ae),l(S),l(ne),l(ie),l(pe),l(x),l(re),l(B),l(oe),l(me),l(y),l(Me),l(ue),l(E),l(fe),l(ce),l(F)),l(g),f(U,e),f(h,e),f(T,e),f(C,e),f(b,e),f(v,e),f(J,e),f(k,e),f(A,e),f(Z,e),f(V,e),f(H,e),f(q,e)}}}const qe='{"title":"Using LiteLLM as Backend","local":"using-litellm-as-backend","sections":[{"title":"Basic Usage","local":"basic-usage","sections":[],"depth":2},{"title":"Using a Configuration File","local":"using-a-configuration-file","sections":[],"depth":2},{"title":"Supported Providers","local":"supported-providers","sections":[{"title":"Cloud Providers","local":"cloud-providers","sections":[],"depth":3},{"title":"Local/On-Premise","local":"localon-premise","sections":[],"depth":3}],"depth":2},{"title":"Using with Local Models","local":"using-with-local-models","sections":[{"title":"VLLM Server","local":"vllm-server","sections":[],"depth":3}],"depth":2}],"depth":1}';function We(de){return Ae(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Xe extends Ze{constructor(g){super(),xe(this,g,We,Ee,ke,{})}}export{Xe as component};