lighteval/pr_706/en/_app/immutable/nodes/23.d52ff2cb.js

import{s as k,n as Y,o as R}from"../chunks/scheduler.7da89386.js";import{S as C,i as F,g as w,s as c,r as b,A as N,h as j,f as e,c as r,j as E,u as A,x as V,k as W,y as x,a as s,v as f,d as Z,t as g,w as G}from"../chunks/index.20910acc.js";import{C as z}from"../chunks/CodeBlock.143bd81e.js";import{H as S,E as H}from"../chunks/index.3af8f81c.js";function P(_){let t,m,T,h,n,y,M,v=`Lighteval can be used from a custom python script. To evaluate a model you will need to set up an
<a href="/docs/lighteval/pr_706/en/package_reference/evaluation_tracker#lighteval.logging.evaluation_tracker.EvaluationTracker">EvaluationTracker</a>, <a href="/docs/lighteval/pr_706/en/package_reference/pipeline#lighteval.pipeline.PipelineParameters">PipelineParameters</a>,
a <a href="package_reference/models"><code>model</code></a> or a <a href="package_reference/model_config"><code>model_config</code></a>,
and a <a href="/docs/lighteval/pr_706/en/package_reference/pipeline#lighteval.pipeline.Pipeline">Pipeline</a>.`,I,J,X="After that, simply run the pipeline and save the results.",d,p,u,i,U,o,B;return n=new S({props:{title:"Using the Python API",local:"using-the-python-api",headingTag:"h1"}}),p=new z({props:{code:"aW1wb3J0JTIwbGlnaHRldmFsJTBBZnJvbSUyMGxpZ2h0ZXZhbC5sb2dnaW5nLmV2YWx1YXRpb25fdHJhY2tlciUyMGltcG9ydCUyMEV2YWx1YXRpb25UcmFja2VyJTBBZnJvbSUyMGxpZ2h0ZXZhbC5tb2RlbHMudmxsbS52bGxtX21vZGVsJTIwaW1wb3J0JTIwVkxMTU1vZGVsQ29uZmlnJTBBZnJvbSUyMGxpZ2h0ZXZhbC5waXBlbGluZSUyMGltcG9ydCUyMFBhcmFsbGVsaXNtTWFuYWdlciUyQyUyMFBpcGVsaW5lJTJDJTIwUGlwZWxpbmVQYXJhbWV0ZXJzJTBBZnJvbSUyMGxpZ2h0ZXZhbC51dGlscy51dGlscyUyMGltcG9ydCUyMEVudkNvbmZpZyUwQWZyb20lMjBsaWdodGV2YWwudXRpbHMuaW1wb3J0cyUyMGltcG9ydCUyMGlzX2FjY2VsZXJhdGVfYXZhaWxhYmxlJTBBJTBBaWYlMjBpc19hY2NlbGVyYXRlX2F2YWlsYWJsZSgpJTNBJTBBJTIwJTIwJTIwJTIwZnJvbSUyMGRhdGV0aW1lJTIwaW1wb3J0JTIwdGltZWRlbHRhJTBBJTIwJTIwJTIwJTIwZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUyQyUyMEluaXRQcm9jZXNzR3JvdXBLd2FyZ3MlMEElMjAlMjAlMjAlMjBhY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKGt3YXJnc19oYW5kbGVycyUzRCU1QkluaXRQcm9jZXNzR3JvdXBLd2FyZ3ModGltZW91dCUzRHRpbWVkZWx0YShzZWNvbmRzJTNEMzAwMCkpJTVEKSUwQWVsc2UlM0ElMEElMjAlMjAlMjAlMjBhY2NlbGVyYXRvciUyMCUzRCUyME5vbmUlMEElMEFkZWYlMjBtYWluKCklM0ElMEElMjAlMjAlMjAlMjBldmFsdWF0aW9uX3RyYWNrZXIlMjAlM0QlMjBFdmFsdWF0aW9uVHJhY2tlciglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIyLiUyRnJlc3VsdHMlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzYXZlX2RldGFpbHMlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaHViX3Jlc3VsdHNfb3JnJTNEJTIyeW91ciUyMHVzZXIlMjBuYW1lJTIyJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMHBpcGVsaW5lX3BhcmFtcyUyMCUzRCUyMFBpcGVsaW5lUGFyYW1ldGVycyglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYXVuY2hlcl90eXBlJTNEUGFyYWxsZWxpc21NYW5hZ2VyLkFDQ0VMRVJBVEUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbnZfY29uZmlnJTNERW52Q29uZmlnKGNhY2hlX2RpciUzRCUyMnRtcCUyRiUyMiklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjdXN0b21fdGFza19kaXJlY3RvcnklM0ROb25lJTJDJTIwJTIzJTIwaWYlMjB1c2luZyUyMGElMjBjdXN0b20lMjB0YXNrJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwUmVtb3ZlJTIwdGhlJTIwMiUyMHBhcmFtZXRlcnMlMjBiZWxvdyUyMG9uY2UlMjB5b3VyJTIwY29uZmlndXJhdGlvbiUyMGlzJTIwdGVzdGVkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3ZlcnJpZGVfYmF0Y2hfc2l6ZSUzRDElMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXhfc2FtcGxlcyUzRDEwJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMG1vZGVsX2NvbmZpZyUyMCUzRCUyMFZMTE1Nb2RlbENvbmZpZyglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTNEJTIySHVnZ2luZ0ZhY2VINCUyRnplcGh5ci03Yi1iZXRhJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZHR5cGUlM0QlMjJmbG9hdDE2JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdXNlX2NoYXRfdGVtcGxhdGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMHRhc2slMjAlM0QlMjAlMjJoZWxtJTdDbW1sdSU3QzUlN0MxJTIyJTBBJTBBJTIwJTIwJTIwJTIwcGlwZWxpbmUlMjAlM0QlMjBQaXBlbGluZSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0YXNrcyUzRHRhc2slMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwaXBlbGluZV9wYXJhbWV0ZXJzJTNEcGlwZWxpbmVfcGFyYW1zJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXZhbHVhdGlvbl90cmFja2VyJTNEZXZhbHVhdGlvbl90cmFja2VyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbW9kZWxfY29uZmlnJTNEbW9kZWxfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMHBpcGVsaW5lLmV2YWx1YXRlKCklMEElMjAlMjAlMjAlMjBwaXBlbGluZS5zYXZlX2FuZF9wdXNoX3Jlc3VsdHMoKSUwQSUyMCUyMCUyMCUyMHBpcGVsaW5lLnNob3dfcmVzdWx0cygpJTBBJTBBaWYlMjBfX25hbWVfXyUyMCUzRCUzRCUyMCUyMl9fbWFpbl9fJTIyJTNBJTBBJTIwJTIwJTIwJTIwbWFpbigp",highlighted:`<span class="hljs-keyword">import</span> lighteval
<span class="hljs-keyword">from</span> lighteval.logging.evaluation_tracker <span class="hljs-keyword">import</span> EvaluationTracker
<span class="hljs-keyword">from</span> lighteval.models.vllm.vllm_model <span class="hljs-keyword">import</span> VLLMModelConfig
<span class="hljs-keyword">from</span> lighteval.pipeline <span class="hljs-keyword">import</span> ParallelismManager, Pipeline, PipelineParameters
<span class="hljs-keyword">from</span> lighteval.utils.utils <span class="hljs-keyword">import</span> EnvConfig
<span class="hljs-keyword">from</span> lighteval.utils.imports <span class="hljs-keyword">import</span> is_accelerate_available

<span class="hljs-keyword">if</span> is_accelerate_available():
    <span class="hljs-keyword">from</span> datetime <span class="hljs-keyword">import</span> timedelta
    <span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator, InitProcessGroupKwargs
    accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=<span class="hljs-number">3000</span>))])
<span class="hljs-keyword">else</span>:
    accelerator = <span class="hljs-literal">None</span>

<span class="hljs-keyword">def</span> <span class="hljs-title function_">main</span>():
    evaluation_tracker = EvaluationTracker(
        output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
        save_details=<span class="hljs-literal">True</span>,
        push_to_hub=<span class="hljs-literal">True</span>,
        hub_results_org=<span class="hljs-string">&quot;your user name&quot;</span>,
    )

    pipeline_params = PipelineParameters(
        launcher_type=ParallelismManager.ACCELERATE,
        env_config=EnvConfig(cache_dir=<span class="hljs-string">&quot;tmp/&quot;</span>),
        custom_task_directory=<span class="hljs-literal">None</span>, <span class="hljs-comment"># if using a custom task</span>
        <span class="hljs-comment"># Remove the 2 parameters below once your configuration is tested</span>
        override_batch_size=<span class="hljs-number">1</span>,
        max_samples=<span class="hljs-number">10</span>
    )

    model_config = VLLMModelConfig(
            model_name=<span class="hljs-string">&quot;HuggingFaceH4/zephyr-7b-beta&quot;</span>,
            dtype=<span class="hljs-string">&quot;float16&quot;</span>,
            use_chat_template=<span class="hljs-literal">True</span>,
    )

    task = <span class="hljs-string">&quot;helm|mmlu|5|1&quot;</span>

    pipeline = Pipeline(
        tasks=task,
        pipeline_parameters=pipeline_params,
        evaluation_tracker=evaluation_tracker,
        model_config=model_config,
    )

    pipeline.evaluate()
    pipeline.save_and_push_results()
    pipeline.show_results()

<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    main()`,wrap:!1}}),i=new H({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/using-the-python-api.mdx"}}),{c(){t=w("meta"),m=c(),T=w("p"),h=c(),b(n.$$.fragment),y=c(),M=w("p"),M.innerHTML=v,I=c(),J=w("p"),J.textContent=X,d=c(),b(p.$$.fragment),u=c(),b(i.$$.fragment),U=c(),o=w("p"),this.h()},l(l){const a=N("svelte-u9bgzb",document.head);t=j(a,"META",{name:!0,content:!0}),a.forEach(e),m=r(l),T=j(l,"P",{}),E(T).forEach(e),h=r(l),A(n.$$.fragment,l),y=r(l),M=j(l,"P",{"data-svelte-h":!0}),V(M)!=="svelte-y37ruq"&&(M.innerHTML=v),I=r(l),J=j(l,"P",{"data-svelte-h":!0}),V(J)!=="svelte-cbze7g"&&(J.textContent=X),d=r(l),A(p.$$.fragment,l),u=r(l),A(i.$$.fragment,l),U=r(l),o=j(l,"P",{}),E(o).forEach(e),this.h()},h(){W(t,"name","hf:doc:metadata"),W(t,"content",Q)},m(l,a){x(document.head,t),s(l,m,a),s(l,T,a),s(l,h,a),f(n,l,a),s(l,y,a),s(l,M,a),s(l,I,a),s(l,J,a),s(l,d,a),f(p,l,a),s(l,u,a),f(i,l,a),s(l,U,a),s(l,o,a),B=!0},p:Y,i(l){B||(Z(n.$$.fragment,l),Z(p.$$.fragment,l),Z(i.$$.fragment,l),B=!0)},o(l){g(n.$$.fragment,l),g(p.$$.fragment,l),g(i.$$.fragment,l),B=!1},d(l){l&&(e(m),e(T),e(h),e(y),e(M),e(I),e(J),e(d),e(u),e(U),e(o)),e(t),G(n,l),G(p,l),G(i,l)}}}const Q='{"title":"Using the Python API","local":"using-the-python-api","sections":[],"depth":1}';function $(_){return R(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class O extends C{constructor(t){super(),F(this,t,$,P,k,{})}}export{O as component};