lighteval/pr_1221/en/_app/immutable/nodes/25.4eb65eaa.js

import{s as xe,n as Be,o as Ve}from"../chunks/scheduler.3a17fb72.js";import{S as Ee,i as Se,e as i,s as a,c as p,h as Ze,a as r,d as l,b as n,f as ke,g as o,j as $,k as ye,l as Ge,m as s,n as m,t as M,o as u,p as f}from"../chunks/index.093f8863.js";import{C as He,H as y,E as We}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.5e7ea2bd.js";import{C as Te}from"../chunks/CodeBlock.09235327.js";function qe(he){let c,F,q,P,T,X,h,N,w,we=`Lighteval allows you to use LiteLLM as a backend, enabling you to call all LLM APIs
using the OpenAI format. LiteLLM supports various providers including Bedrock, Hugging Face, Vertex AI, Together AI, Azure,
OpenAI, Groq, and many others.`,Q,g,je='<p>Documentation for available APIs and compatible endpoints can be found <a href="https://docs.litellm.ai/docs/" rel="nofollow">here</a>.</p>',z,j,O,U,Y,L,D,b,Ue=`LiteLLM allows generation with any OpenAI-compatible endpoint. For example, you
can evaluate a model running on a local VLLM server.`,K,v,Le="To do so, you will need to use a configuration file like this:",ee,J,te,C,le,I,be="LiteLLM supports a wide range of LLM providers:",se,_,ae,A,ve='all cloud providers can be found in the <a href="https://docs.litellm.ai/docs/providers" rel="nofollow">litellm documentation</a>.',ne,k,ie,x,Je="<li><strong>VLLM</strong>: Local VLLM servers</li> <li><strong>Hugging Face</strong>: Local Hugging Face models</li> <li><strong>Custom endpoints</strong>: Any OpenAI-compatible API</li>",pe,B,re,V,oe,E,Ce="To use with a local VLLM server:",me,S,Ie="<li>Start your VLLM server:</li>",Me,Z,ue,d,_e="<li>Configure LiteLLM to use the local server:</li>",fe,G,$e,H,Ae='For more detailed error handling and debugging, refer to the <a href="https://docs.litellm.ai/docs/" rel="nofollow">LiteLLM documentation</a>.',ce,W,ge,R,de;return T=new He({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),h=new y({props:{title:"Using LiteLLM as Backend",local:"using-litellm-as-backend",headingTag:"h1"}}),j=new y({props:{title:"Basic Usage",local:"basic-usage",headingTag:"h2"}}),U=new Te({props:{code:"bGlnaHRldmFsJTIwZW5kcG9pbnQlMjBsaXRlbGxtJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIycHJvdmlkZXIlM0RvcGVuYWklMkNtb2RlbF9uYW1lJTNEZ3B0LTMuNS10dXJibyUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMGdzbThr",highlighted:`lighteval endpoint litellm \\
    <span class="hljs-string">&quot;provider=openai,model_name=gpt-3.5-turbo&quot;</span> \\
    gsm8k`,wrap:!1}}),L=new y({props:{title:"Using a Configuration File",local:"using-a-configuration-file",headingTag:"h2"}}),J=new Te({props:{code:"bW9kZWxfcGFyYW1ldGVycyUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0ElMjAlMjJvcGVuYWklMkZkZWVwc2Vlay1haSUyRkRlZXBTZWVrLVIxLURpc3RpbGwtUXdlbi0zMkIlMjIlMEElMjAlMjAlMjAlMjBiYXNlX3VybCUzQSUyMCUyMlVSTF9PRl9USEVfRU5EUE9JTlRfWU9VX1dBTlRfVE9fVVNFJTIyJTBBJTIwJTIwJTIwJTIwYXBpX2tleSUzQSUyMCUyMiUyMiUyMCUyMyUyMFJlbW92ZSUyMG9yJTIwa2VlcCUyMGVtcHR5JTIwYXMlMjBuZWVkZWQlMEElMjAlMjAlMjAlMjBnZW5lcmF0aW9uX3BhcmFtZXRlcnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzQSUyMDAuNSUwQSUyMCUyMCUyMCUyMCUyMCUyMG1heF9uZXdfdG9rZW5zJTNBJTIwMjU2JTBBJTIwJTIwJTIwJTIwJTIwJTIwc3RvcF90b2tlbnMlM0ElMjAlNUIlMjIlMjIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjB0b3BfcCUzQSUyMDAuOSUwQSUyMCUyMCUyMCUyMCUyMCUyMHNlZWQlM0ElMjAwJTBBJTIwJTIwJTIwJTIwJTIwJTIwcmVwZXRpdGlvbl9wZW5hbHR5JTNBJTIwMS4wJTBBJTIwJTIwJTIwJTIwJTIwJTIwZnJlcXVlbmN5X3BlbmFsdHklM0ElMjAwLjA=",highlighted:`<span class="hljs-attr">model_parameters:</span>
    <span class="hljs-attr">model_name:</span> <span class="hljs-string">&quot;openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B&quot;</span>
    <span class="hljs-attr">base_url:</span> <span class="hljs-string">&quot;URL_OF_THE_ENDPOINT_YOU_WANT_TO_USE&quot;</span>
    <span class="hljs-attr">api_key:</span> <span class="hljs-string">&quot;&quot;</span> <span class="hljs-comment"># Remove or keep empty as needed</span>
    <span class="hljs-attr">generation_parameters:</span>
      <span class="hljs-attr">temperature:</span> <span class="hljs-number">0.5</span>
      <span class="hljs-attr">max_new_tokens:</span> <span class="hljs-number">256</span>
      <span class="hljs-attr">stop_tokens:</span> [<span class="hljs-string">&quot;&quot;</span>]
      <span class="hljs-attr">top_p:</span> <span class="hljs-number">0.9</span>
      <span class="hljs-attr">seed:</span> <span class="hljs-number">0</span>
      <span class="hljs-attr">repetition_penalty:</span> <span class="hljs-number">1.0</span>
      <span class="hljs-attr">frequency_penalty:</span> <span class="hljs-number">0.0</span>`,wrap:!1}}),C=new y({props:{title:"Supported Providers",local:"supported-providers",headingTag:"h2"}}),_=new y({props:{title:"Cloud Providers",local:"cloud-providers",headingTag:"h3"}}),k=new y({props:{title:"Local/On-Premise",local:"localon-premise",headingTag:"h3"}}),B=new y({props:{title:"Using with Local Models",local:"using-with-local-models",headingTag:"h2"}}),V=new y({props:{title:"VLLM Server",local:"vllm-server",headingTag:"h3"}}),Z=new Te({props:{code:"dmxsbSUyMHNlcnZlJTIwSHVnZ2luZ0ZhY2VINCUyRnplcGh5ci03Yi1iZXRhJTIwLS1ob3N0JTIwMC4wLjAuMCUyMC0tcG9ydCUyMDgwMDA=",highlighted:"vllm serve HuggingFaceH4/zephyr-7b-beta --host 0.0.0.0 --port 8000",wrap:!1}}),G=new Te({props:{code:"bW9kZWxfcGFyYW1ldGVycyUzQSUwQSUyMCUyMCUyMCUyMHByb3ZpZGVyJTNBJTIwJTIyaG9zdGVkX3ZsbG0lMjIlMEElMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTNBJTIwJTIyaG9zdGVkX3ZsbG0lMkZIdWdnaW5nRmFjZUg0JTJGemVwaHlyLTdiLWJldGElMjIlMEElMjAlMjAlMjAlMjBiYXNlX3VybCUzQSUyMCUyMmh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDAwJTJGdjElMjIlMEElMjAlMjAlMjAlMjBhcGlfa2V5JTNBJTIwJTIyJTIy",highlighted:`<span class="hljs-attr">model_parameters:</span>
    <span class="hljs-attr">provider:</span> <span class="hljs-string">&quot;hosted_vllm&quot;</span>
    <span class="hljs-attr">model_name:</span> <span class="hljs-string">&quot;hosted_vllm/HuggingFaceH4/zephyr-7b-beta&quot;</span>
    <span class="hljs-attr">base_url:</span> <span class="hljs-string">&quot;http://localhost:8000/v1&quot;</span>
    <span class="hljs-attr">api_key:</span> <span class="hljs-string">&quot;&quot;</span>`,wrap:!1}}),W=new We({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/use-litellm-as-backend.mdx"}}),{c(){c=i("meta"),F=a(),q=i("p"),P=a(),p(T.$$.fragment),X=a(),p(h.$$.fragment),N=a(),w=i("p"),w.textContent=we,Q=a(),g=i("blockquote"),g.innerHTML=je,z=a(),p(j.$$.fragment),O=a(),p(U.$$.fragment),Y=a(),p(L.$$.fragment),D=a(),b=i("p"),b.textContent=Ue,K=a(),v=i("p"),v.textContent=Le,ee=a(),p(J.$$.fragment),te=a(),p(C.$$.fragment),le=a(),I=i("p"),I.textContent=be,se=a(),p(_.$$.fragment),ae=a(),A=i("p"),A.innerHTML=ve,ne=a(),p(k.$$.fragment),ie=a(),x=i("ul"),x.innerHTML=Je,pe=a(),p(B.$$.fragment),re=a(),p(V.$$.fragment),oe=a(),E=i("p"),E.textContent=Ce,me=a(),S=i("ol"),S.innerHTML=Ie,Me=a(),p(Z.$$.fragment),ue=a(),d=i("ol"),d.innerHTML=_e,fe=a(),p(G.$$.fragment),$e=a(),H=i("p"),H.innerHTML=Ae,ce=a(),p(W.$$.fragment),ge=a(),R=i("p"),this.h()},l(e){const t=Ze("svelte-u9bgzb",document.head);c=r(t,"META",{name:!0,content:!0}),t.forEach(l),F=n(e),q=r(e,"P",{}),ke(q).forEach(l),P=n(e),o(T.$$.fragment,e),X=n(e),o(h.$$.fragment,e),N=n(e),w=r(e,"P",{"data-svelte-h":!0}),$(w)!=="svelte-a1o746"&&(w.textContent=we),Q=n(e),g=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),$(g)!=="svelte-tnom0v"&&(g.innerHTML=je),z=n(e),o(j.$$.fragment,e),O=n(e),o(U.$$.fragment,e),Y=n(e),o(L.$$.fragment,e),D=n(e),b=r(e,"P",{"data-svelte-h":!0}),$(b)!=="svelte-3l1m4a"&&(b.textContent=Ue),K=n(e),v=r(e,"P",{"data-svelte-h":!0}),$(v)!=="svelte-pxnxuz"&&(v.textContent=Le),ee=n(e),o(J.$$.fragment,e),te=n(e),o(C.$$.fragment,e),le=n(e),I=r(e,"P",{"data-svelte-h":!0}),$(I)!=="svelte-10eon5u"&&(I.textContent=be),se=n(e),o(_.$$.fragment,e),ae=n(e),A=r(e,"P",{"data-svelte-h":!0}),$(A)!=="svelte-9dfpjp"&&(A.innerHTML=ve),ne=n(e),o(k.$$.fragment,e),ie=n(e),x=r(e,"UL",{"data-svelte-h":!0}),$(x)!=="svelte-1craf7w"&&(x.innerHTML=Je),pe=n(e),o(B.$$.fragment,e),re=n(e),o(V.$$.fragment,e),oe=n(e),E=r(e,"P",{"data-svelte-h":!0}),$(E)!=="svelte-tacy60"&&(E.textContent=Ce),me=n(e),S=r(e,"OL",{"data-svelte-h":!0}),$(S)!=="svelte-xokqka"&&(S.innerHTML=Ie),Me=n(e),o(Z.$$.fragment,e),ue=n(e),d=r(e,"OL",{start:!0,"data-svelte-h":!0}),$(d)!=="svelte-1ujc1uy"&&(d.innerHTML=_e),fe=n(e),o(G.$$.fragment,e),$e=n(e),H=r(e,"P",{"data-svelte-h":!0}),$(H)!=="svelte-fyarqi"&&(H.innerHTML=Ae),ce=n(e),o(W.$$.fragment,e),ge=n(e),R=r(e,"P",{}),ke(R).forEach(l),this.h()},h(){ye(c,"name","hf:doc:metadata"),ye(c,"content",Re),ye(g,"class","tip"),ye(d,"start","2")},m(e,t){Ge(document.head,c),s(e,F,t),s(e,q,t),s(e,P,t),m(T,e,t),s(e,X,t),m(h,e,t),s(e,N,t),s(e,w,t),s(e,Q,t),s(e,g,t),s(e,z,t),m(j,e,t),s(e,O,t),m(U,e,t),s(e,Y,t),m(L,e,t),s(e,D,t),s(e,b,t),s(e,K,t),s(e,v,t),s(e,ee,t),m(J,e,t),s(e,te,t),m(C,e,t),s(e,le,t),s(e,I,t),s(e,se,t),m(_,e,t),s(e,ae,t),s(e,A,t),s(e,ne,t),m(k,e,t),s(e,ie,t),s(e,x,t),s(e,pe,t),m(B,e,t),s(e,re,t),m(V,e,t),s(e,oe,t),s(e,E,t),s(e,me,t),s(e,S,t),s(e,Me,t),m(Z,e,t),s(e,ue,t),s(e,d,t),s(e,fe,t),m(G,e,t),s(e,$e,t),s(e,H,t),s(e,ce,t),m(W,e,t),s(e,ge,t),s(e,R,t),de=!0},p:Be,i(e){de||(M(T.$$.fragment,e),M(h.$$.fragment,e),M(j.$$.fragment,e),M(U.$$.fragment,e),M(L.$$.fragment,e),M(J.$$.fragment,e),M(C.$$.fragment,e),M(_.$$.fragment,e),M(k.$$.fragment,e),M(B.$$.fragment,e),M(V.$$.fragment,e),M(Z.$$.fragment,e),M(G.$$.fragment,e),M(W.$$.fragment,e),de=!0)},o(e){u(T.$$.fragment,e),u(h.$$.fragment,e),u(j.$$.fragment,e),u(U.$$.fragment,e),u(L.$$.fragment,e),u(J.$$.fragment,e),u(C.$$.fragment,e),u(_.$$.fragment,e),u(k.$$.fragment,e),u(B.$$.fragment,e),u(V.$$.fragment,e),u(Z.$$.fragment,e),u(G.$$.fragment,e),u(W.$$.fragment,e),de=!1},d(e){e&&(l(F),l(q),l(P),l(X),l(N),l(w),l(Q),l(g),l(z),l(O),l(Y),l(D),l(b),l(K),l(v),l(ee),l(te),l(le),l(I),l(se),l(ae),l(A),l(ne),l(ie),l(x),l(pe),l(re),l(oe),l(E),l(me),l(S),l(Me),l(ue),l(d),l(fe),l($e),l(H),l(ce),l(ge),l(R)),l(c),f(T,e),f(h,e),f(j,e),f(U,e),f(L,e),f(J,e),f(C,e),f(_,e),f(k,e),f(B,e),f(V,e),f(Z,e),f(G,e),f(W,e)}}}const Re='{"title":"Using LiteLLM as Backend","local":"using-litellm-as-backend","sections":[{"title":"Basic Usage","local":"basic-usage","sections":[],"depth":2},{"title":"Using a Configuration File","local":"using-a-configuration-file","sections":[],"depth":2},{"title":"Supported Providers","local":"supported-providers","sections":[{"title":"Cloud Providers","local":"cloud-providers","sections":[],"depth":3},{"title":"Local/On-Premise","local":"localon-premise","sections":[],"depth":3}],"depth":2},{"title":"Using with Local Models","local":"using-with-local-models","sections":[{"title":"VLLM Server","local":"vllm-server","sections":[],"depth":3}],"depth":2}],"depth":1}';function Fe(he){return Ve(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ze extends Ee{constructor(c){super(),Se(this,c,Fe,qe,xe,{})}}export{ze as component};