lighteval/pr_476/en/_app/immutable/nodes/7.aa7465dd.js

import{s as ol,o as pl,n as ml}from"../chunks/scheduler.7da89386.js";import{S as cl,i as rl,g as i,s as a,r as c,A as Ml,h as o,f as t,c as n,j as ae,u as r,x as m,k as Qe,y,a as s,v as M,d,t as h,w as u}from"../chunks/index.20910acc.js";import{T as dl}from"../chunks/Tip.53e22153.js";import{C as ne}from"../chunks/CodeBlock.143bd81e.js";import{H as g,E as hl}from"../chunks/getInferenceSnippets.cb95a28d.js";function ul(oe){let p,b="You can find a complete example of a custom model implementation in <code>examples/custom_models/google_translate_model.py</code>.";return{c(){p=i("p"),p.innerHTML=b},l(j){p=o(j,"P",{"data-svelte-h":!0}),m(p)!=="svelte-1s5ktfo"&&(p.innerHTML=b)},m(j,ee){s(j,p,ee)},p:ml,d(j){j&&t(p)}}}function yl(oe){let p,b,j,ee,U,pe,Z,ze=`Lighteval allows you to evaluate custom model implementations by creating a custom model class that inherits from <code>LightevalModel</code>.
This is useful when you want to evaluate models that aren’t directly supported by the standard backends and providers (transformers, vllm, etc), or
if you want to add your own pre/post processing.`,me,C,ce,v,Fe="<li>Create a Python file containing your custom model implementation. The model must inherit from <code>LightevalModel</code> and implement all required methods.</li>",re,I,He="Here’s a basic example:",Me,$,de,f,Se="<li>The custom model file should contain exactly one class that inherits from <code>LightevalModel</code>. This class will be automatically detected and instantiated when loading the model.</li>",he,J,ue,G,ye,W,qe="You can evaluate your custom model using either the command line interface or the Python API.",je,B,Te,_,we,k,Pe="The command takes three required arguments:",fe,x,De="<li>The model name (used for tracking in results/logs)</li> <li>The path to your model implementation file</li> <li>The tasks to evaluate on (same format as other backends)</li>",Je,E,ge,A,be,V,Ue,Y,Ke="Your custom model must implement these core methods:",Ze,X,Oe="<li><code>greedy_until</code>: For generating text until a stop sequence or max tokens is reached - this is used for generative evaluations</li> <li><code>loglikelihood</code>: For computing log probabilities of specific continuations - this is used for multiple choice logprob evaluations</li> <li><code>loglikelihood_rolling</code>: For computing rolling log probabilities of sequences - this is used for perplexity metrics</li>",Ce,R,el="See the <code>LightevalModel</code> base class documentation for detailed method signatures and requirements.",ve,N,Ie,L,ll=`Lighteval includes a caching system that can significantly speed up evaluations by storing and reusing model predictions.
To enable caching in your custom model:`,$e,T,Q,le,tl="<strong>Import caching components</strong>:",Ye,z,Xe,F,te,sl="<strong>Initialize cache in constructor</strong>:",Re,H,Ne,S,se,al="<strong>Add cache decorators</strong> to your prediction methods:",Le,q,Ge,P,nl='For detailed information about the caching system, see the <a href="./caching.mdx">Caching Documentation</a>.',We,D,Be,K,il="<li><p><strong>Error Handling</strong>: Implement robust error handling in your model methods to gracefully handle edge cases.</p></li> <li><p><strong>Batching</strong>: Consider implementing efficient batching in your model methods to improve performance.</p></li> <li><p><strong>Documentation</strong>: Add clear docstrings to your model class and methods explaining any specific requirements or limitations.</p></li> <li><p><strong>Caching</strong>: Enable caching to speed up repeated evaluations and development iterations.</p></li>",_e,O,ke,ie,xe;return U=new g({props:{title:"Custom Model",local:"custom-model",headingTag:"h1"}}),C=new g({props:{title:"Creating a Custom Model",local:"creating-a-custom-model",headingTag:"h2"}}),$=new ne({props:{code:"ZnJvbSUyMHR5cGluZyUyMGltcG9ydCUyMExpc3QlMEFmcm9tJTIwbGlnaHRldmFsLm1vZGVscy5hYnN0cmFjdF9tb2RlbCUyMGltcG9ydCUyMExpZ2h0ZXZhbE1vZGVsJTBBZnJvbSUyMGxpZ2h0ZXZhbC5tb2RlbHMubW9kZWxfb3V0cHV0JTIwaW1wb3J0JTIwTW9kZWxSZXNwb25zZSUwQWZyb20lMjBsaWdodGV2YWwudGFza3MucmVxdWVzdHMlMjBpbXBvcnQlMjBEb2MlMEFmcm9tJTIwbGlnaHRldmFsLnV0aWxzLmNhY2hlX21hbmFnZW1lbnQlMjBpbXBvcnQlMjBTYW1wbGVDYWNoZSUyQyUyMGNhY2hlZCUwQSUwQWNsYXNzJTIwTXlDdXN0b21Nb2RlbChMaWdodGV2YWxNb2RlbCklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmJTJDJTIwY29uZmlnKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN1cGVyKCkuX19pbml0X18oY29uZmlnKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEluaXRpYWxpemUlMjB5b3VyJTIwbW9kZWwlMjBoZXJlLi4uJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwRW5hYmxlJTIwY2FjaGluZyUyMChyZWNvbW1lbmRlZCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLl9jYWNoZSUyMCUzRCUyMFNhbXBsZUNhY2hlKGNvbmZpZyklMEElMEElMjAlMjAlMjAlMjAlNDBjYWNoZWQoJTIycHJlZGljdGlvbnMlMjIpJTIwJTIwJTIzJTIwRW5hYmxlJTIwY2FjaGluZyUyMGZvciUyMGJldHRlciUyMHBlcmZvcm1hbmNlJTBBJTIwJTIwJTIwJTIwZGVmJTIwZ3JlZWR5X3VudGlsKHNlbGYlMkMlMjBkb2NzJTNBJTIwTGlzdCU1QkRvYyU1RCklMjAtJTNFJTIwTGlzdCU1Qk1vZGVsUmVzcG9uc2UlNUQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBJbXBsZW1lbnQlMjBnZW5lcmF0aW9uJTIwbG9naWMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwYXNzJTBBJTBBJTIwJTIwJTIwJTIwJTQwY2FjaGVkKCUyMnByZWRpY3Rpb25zJTIyKSUyMCUyMCUyMyUyMEVuYWJsZSUyMGNhY2hpbmclMjBmb3IlMjBiZXR0ZXIlMjBwZXJmb3JtYW5jZSUwQSUyMCUyMCUyMCUyMGRlZiUyMGxvZ2xpa2VsaWhvb2Qoc2VsZiUyQyUyMGRvY3MlM0ElMjBMaXN0JTVCRG9jJTVEKSUyMC0lM0UlMjBMaXN0JTVCTW9kZWxSZXNwb25zZSU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEltcGxlbWVudCUyMGxvZ2xpa2VsaWhvb2QlMjBjb21wdXRhdGlvbiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhc3MlMEElMEElMjAlMjAlMjAlMjAlNDBjYWNoZWQoJTIycHJlZGljdGlvbnMlMjIpJTIwJTIwJTIzJTIwRW5hYmxlJTIwY2FjaGluZyUyMGZvciUyMGJldHRlciUyMHBlcmZvcm1hbmNlJTBBJTIwJTIwJTIwJTIwZGVmJTIwbG9nbGlrZWxpaG9vZF9yb2xsaW5nKHNlbGYlMkMlMjBkb2NzJTNBJTIwTGlzdCU1QkRvYyU1RCklMjAtJTNFJTIwTGlzdCU1Qk1vZGVsUmVzcG9uc2UlNUQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBJbXBsZW1lbnQlMjByb2xsaW5nJTIwbG9nbGlrZWxpaG9vZCUyMGNvbXB1dGF0aW9uJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGFzcw==",highlighted:`<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">List</span>
<span class="hljs-keyword">from</span> lighteval.models.abstract_model <span class="hljs-keyword">import</span> LightevalModel
<span class="hljs-keyword">from</span> lighteval.models.model_output <span class="hljs-keyword">import</span> ModelResponse
<span class="hljs-keyword">from</span> lighteval.tasks.requests <span class="hljs-keyword">import</span> Doc
<span class="hljs-keyword">from</span> lighteval.utils.cache_management <span class="hljs-keyword">import</span> SampleCache, cached

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyCustomModel</span>(<span class="hljs-title class_ inherited__">LightevalModel</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
        <span class="hljs-built_in">super</span>().__init__(config)
        <span class="hljs-comment"># Initialize your model here...</span>

        <span class="hljs-comment"># Enable caching (recommended)</span>
        self._cache = SampleCache(config)

<span class="hljs-meta">    @cached(<span class="hljs-params"><span class="hljs-string">&quot;predictions&quot;</span></span>)  </span><span class="hljs-comment"># Enable caching for better performance</span>
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">greedy_until</span>(<span class="hljs-params">self, docs: <span class="hljs-type">List</span>[Doc]</span>) -&gt; <span class="hljs-type">List</span>[ModelResponse]:
        <span class="hljs-comment"># Implement generation logic</span>
        <span class="hljs-keyword">pass</span>

<span class="hljs-meta">    @cached(<span class="hljs-params"><span class="hljs-string">&quot;predictions&quot;</span></span>)  </span><span class="hljs-comment"># Enable caching for better performance</span>
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">loglikelihood</span>(<span class="hljs-params">self, docs: <span class="hljs-type">List</span>[Doc]</span>) -&gt; <span class="hljs-type">List</span>[ModelResponse]:
        <span class="hljs-comment"># Implement loglikelihood computation</span>
        <span class="hljs-keyword">pass</span>

<span class="hljs-meta">    @cached(<span class="hljs-params"><span class="hljs-string">&quot;predictions&quot;</span></span>)  </span><span class="hljs-comment"># Enable caching for better performance</span>
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">loglikelihood_rolling</span>(<span class="hljs-params">self, docs: <span class="hljs-type">List</span>[Doc]</span>) -&gt; <span class="hljs-type">List</span>[ModelResponse]:
        <span class="hljs-comment"># Implement rolling loglikelihood computation</span>
        <span class="hljs-keyword">pass</span>`,wrap:!1}}),J=new dl({props:{warning:!1,$$slots:{default:[ul]},$$scope:{ctx:oe}}}),G=new g({props:{title:"Running the Evaluation",local:"running-the-evaluation",headingTag:"h2"}}),B=new g({props:{title:"Using the Command Line",local:"using-the-command-line",headingTag:"h3"}}),_=new ne({props:{code:"bGlnaHRldmFsJTIwY3VzdG9tJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIyZ29vZ2xlLXRyYW5zbGF0ZSUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMmV4YW1wbGVzJTJGY3VzdG9tX21vZGVscyUyRmdvb2dsZV90cmFuc2xhdGVfbW9kZWwucHklMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAlMjJsaWdodGV2YWwlN0N3bXQyMCUzQWZyLWRlJTdDMCU3QzAlMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1heC1zYW1wbGVzJTIwMTA=",highlighted:`lighteval custom \\
    <span class="hljs-string">&quot;google-translate&quot;</span> \\
    <span class="hljs-string">&quot;examples/custom_models/google_translate_model.py&quot;</span> \\
    <span class="hljs-string">&quot;lighteval|wmt20:fr-de|0|0&quot;</span> \\
    --max-samples 10`,wrap:!1}}),E=new g({props:{title:"Using the Python API",local:"using-the-python-api",headingTag:"h3"}}),A=new ne({props:{code:"ZnJvbSUyMGxpZ2h0ZXZhbC5sb2dnaW5nLmV2YWx1YXRpb25fdHJhY2tlciUyMGltcG9ydCUyMEV2YWx1YXRpb25UcmFja2VyJTBBZnJvbSUyMGxpZ2h0ZXZhbC5tb2RlbHMuY3VzdG9tLmN1c3RvbV9tb2RlbCUyMGltcG9ydCUyMEN1c3RvbU1vZGVsQ29uZmlnJTBBZnJvbSUyMGxpZ2h0ZXZhbC5waXBlbGluZSUyMGltcG9ydCUyMFBpcGVsaW5lJTJDJTIwUGlwZWxpbmVQYXJhbWV0ZXJzJTBBJTBBJTIzJTIwU2V0JTIwdXAlMjBldmFsdWF0aW9uJTIwdHJhY2tpbmclMEFldmFsdWF0aW9uX3RyYWNrZXIlMjAlM0QlMjBFdmFsdWF0aW9uVHJhY2tlciglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIycmVzdWx0cyUyMiUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfZGV0YWlscyUzRFRydWUlMEEpJTBBJTBBJTIzJTIwQ29uZmlndXJlJTIwdGhlJTIwcGlwZWxpbmUlMEFwaXBlbGluZV9wYXJhbXMlMjAlM0QlMjBQaXBlbGluZVBhcmFtZXRlcnMoJTBBJTIwJTIwJTIwJTIwbGF1bmNoZXJfdHlwZSUzRFBhcmFsbGVsaXNtTWFuYWdlci5DVVNUT00lMkMlMEEpJTBBJTBBJTIzJTIwQ29uZmlndXJlJTIweW91ciUyMGN1c3RvbSUyMG1vZGVsJTBBbW9kZWxfY29uZmlnJTIwJTNEJTIwQ3VzdG9tTW9kZWxDb25maWcoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJteS1jdXN0b20tbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjBtb2RlbF9kZWZpbml0aW9uX2ZpbGVfcGF0aCUzRCUyMnBhdGglMkZ0byUyRm15X21vZGVsLnB5JTIyJTBBKSUwQSUwQSUyMyUyMENyZWF0ZSUyMGFuZCUyMHJ1biUyMHRoZSUyMHBpcGVsaW5lJTBBcGlwZWxpbmUlMjAlM0QlMjBQaXBlbGluZSglMEElMjAlMjAlMjAlMjB0YXNrcyUzRCUyMmxlYWRlcmJvYXJkJTdDdHJ1dGhmdWxxYSUzQW1jJTdDMCU3QzAlMjIlMkMlMEElMjAlMjAlMjAlMjBwaXBlbGluZV9wYXJhbWV0ZXJzJTNEcGlwZWxpbmVfcGFyYW1zJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbl90cmFja2VyJTNEZXZhbHVhdGlvbl90cmFja2VyJTJDJTBBJTIwJTIwJTIwJTIwbW9kZWxfY29uZmlnJTNEbW9kZWxfY29uZmlnJTBBKSUwQSUwQXBpcGVsaW5lLmV2YWx1YXRlKCklMEFwaXBlbGluZS5zYXZlX2FuZF9wdXNoX3Jlc3VsdHMoKQ==",highlighted:`<span class="hljs-keyword">from</span> lighteval.logging.evaluation_tracker <span class="hljs-keyword">import</span> EvaluationTracker
<span class="hljs-keyword">from</span> lighteval.models.custom.custom_model <span class="hljs-keyword">import</span> CustomModelConfig
<span class="hljs-keyword">from</span> lighteval.pipeline <span class="hljs-keyword">import</span> Pipeline, PipelineParameters

<span class="hljs-comment"># Set up evaluation tracking</span>
evaluation_tracker = EvaluationTracker(
    output_dir=<span class="hljs-string">&quot;results&quot;</span>,
    save_details=<span class="hljs-literal">True</span>
)

<span class="hljs-comment"># Configure the pipeline</span>
pipeline_params = PipelineParameters(
    launcher_type=ParallelismManager.CUSTOM,
)

<span class="hljs-comment"># Configure your custom model</span>
model_config = CustomModelConfig(
    model=<span class="hljs-string">&quot;my-custom-model&quot;</span>,
    model_definition_file_path=<span class="hljs-string">&quot;path/to/my_model.py&quot;</span>
)

<span class="hljs-comment"># Create and run the pipeline</span>
pipeline = Pipeline(
    tasks=<span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0|0&quot;</span>,
    pipeline_parameters=pipeline_params,
    evaluation_tracker=evaluation_tracker,
    model_config=model_config
)

pipeline.evaluate()
pipeline.save_and_push_results()`,wrap:!1}}),V=new g({props:{title:"Required Methods",local:"required-methods",headingTag:"h2"}}),N=new g({props:{title:"Enabling Caching (Recommended)",local:"enabling-caching-recommended",headingTag:"h2"}}),z=new ne({props:{code:"ZnJvbSUyMGxpZ2h0ZXZhbC51dGlscy5jYWNoZV9tYW5hZ2VtZW50JTIwaW1wb3J0JTIwU2FtcGxlQ2FjaGUlMkMlMjBjYWNoZWQ=",highlighted:'<span class="hljs-keyword">from</span> lighteval.utils.cache_management <span class="hljs-keyword">import</span> SampleCache, cached',wrap:!1}}),H=new ne({props:{code:"ZGVmJTIwX19pbml0X18oc2VsZiUyQyUyMGNvbmZpZyklM0ElMEElMjAlMjAlMjAlMjAlMjMlMjBZb3VyJTIwaW5pdGlhbGl6YXRpb24lMjBjb2RlLi4uJTBBJTIwJTIwJTIwJTIwc2VsZi5fY2FjaGUlMjAlM0QlMjBTYW1wbGVDYWNoZShjb25maWcp",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
    <span class="hljs-comment"># Your initialization code...</span>
    self._cache = SampleCache(config)`,wrap:!1}}),q=new ne({props:{code:"JTQwY2FjaGVkKCUyMnByZWRpY3Rpb25zJTIyKSUwQWRlZiUyMGdyZWVkeV91bnRpbChzZWxmJTJDJTIwZG9jcyUzQSUyMExpc3QlNUJEb2MlNUQpJTIwLSUzRSUyMExpc3QlNUJNb2RlbFJlc3BvbnNlJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwWW91ciUyMGltcGxlbWVudGF0aW9uLi4u",highlighted:`<span class="hljs-meta">@cached(<span class="hljs-params"><span class="hljs-string">&quot;predictions&quot;</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">greedy_until</span>(<span class="hljs-params">self, docs: <span class="hljs-type">List</span>[Doc]</span>) -&gt; <span class="hljs-type">List</span>[ModelResponse]:
    <span class="hljs-comment"># Your implementation...</span>`,wrap:!1}}),D=new g({props:{title:"Best Practices",local:"best-practices",headingTag:"h2"}}),O=new hl({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/evaluating-a-custom-model.mdx"}}),{c(){p=i("meta"),b=a(),j=i("p"),ee=a(),c(U.$$.fragment),pe=a(),Z=i("p"),Z.innerHTML=ze,me=a(),c(C.$$.fragment),ce=a(),v=i("ol"),v.innerHTML=Fe,re=a(),I=i("p"),I.textContent=He,Me=a(),c($.$$.fragment),de=a(),f=i("ol"),f.innerHTML=Se,he=a(),c(J.$$.fragment),ue=a(),c(G.$$.fragment),ye=a(),W=i("p"),W.textContent=qe,je=a(),c(B.$$.fragment),Te=a(),c(_.$$.fragment),we=a(),k=i("p"),k.textContent=Pe,fe=a(),x=i("ul"),x.innerHTML=De,Je=a(),c(E.$$.fragment),ge=a(),c(A.$$.fragment),be=a(),c(V.$$.fragment),Ue=a(),Y=i("p"),Y.textContent=Ke,Ze=a(),X=i("ul"),X.innerHTML=Oe,Ce=a(),R=i("p"),R.innerHTML=el,ve=a(),c(N.$$.fragment),Ie=a(),L=i("p"),L.textContent=ll,$e=a(),T=i("ol"),Q=i("li"),le=i("p"),le.innerHTML=tl,Ye=a(),c(z.$$.fragment),Xe=a(),F=i("li"),te=i("p"),te.innerHTML=sl,Re=a(),c(H.$$.fragment),Ne=a(),S=i("li"),se=i("p"),se.innerHTML=al,Le=a(),c(q.$$.fragment),Ge=a(),P=i("p"),P.innerHTML=nl,We=a(),c(D.$$.fragment),Be=a(),K=i("ol"),K.innerHTML=il,_e=a(),c(O.$$.fragment),ke=a(),ie=i("p"),this.h()},l(e){const l=Ml("svelte-u9bgzb",document.head);p=o(l,"META",{name:!0,content:!0}),l.forEach(t),b=n(e),j=o(e,"P",{}),ae(j).forEach(t),ee=n(e),r(U.$$.fragment,e),pe=n(e),Z=o(e,"P",{"data-svelte-h":!0}),m(Z)!=="svelte-f5zt17"&&(Z.innerHTML=ze),me=n(e),r(C.$$.fragment,e),ce=n(e),v=o(e,"OL",{"data-svelte-h":!0}),m(v)!=="svelte-5nepfb"&&(v.innerHTML=Fe),re=n(e),I=o(e,"P",{"data-svelte-h":!0}),m(I)!=="svelte-1frc3d"&&(I.textContent=He),Me=n(e),r($.$$.fragment,e),de=n(e),f=o(e,"OL",{start:!0,"data-svelte-h":!0}),m(f)!=="svelte-1uhsqbo"&&(f.innerHTML=Se),he=n(e),r(J.$$.fragment,e),ue=n(e),r(G.$$.fragment,e),ye=n(e),W=o(e,"P",{"data-svelte-h":!0}),m(W)!=="svelte-1icw64v"&&(W.textContent=qe),je=n(e),r(B.$$.fragment,e),Te=n(e),r(_.$$.fragment,e),we=n(e),k=o(e,"P",{"data-svelte-h":!0}),m(k)!=="svelte-9ha93v"&&(k.textContent=Pe),fe=n(e),x=o(e,"UL",{"data-svelte-h":!0}),m(x)!=="svelte-1dz6vvj"&&(x.innerHTML=De),Je=n(e),r(E.$$.fragment,e),ge=n(e),r(A.$$.fragment,e),be=n(e),r(V.$$.fragment,e),Ue=n(e),Y=o(e,"P",{"data-svelte-h":!0}),m(Y)!=="svelte-abp4ax"&&(Y.textContent=Ke),Ze=n(e),X=o(e,"UL",{"data-svelte-h":!0}),m(X)!=="svelte-yt56c1"&&(X.innerHTML=Oe),Ce=n(e),R=o(e,"P",{"data-svelte-h":!0}),m(R)!=="svelte-845vib"&&(R.innerHTML=el),ve=n(e),r(N.$$.fragment,e),Ie=n(e),L=o(e,"P",{"data-svelte-h":!0}),m(L)!=="svelte-1eshig5"&&(L.textContent=ll),$e=n(e),T=o(e,"OL",{});var w=ae(T);Q=o(w,"LI",{});var Ee=ae(Q);le=o(Ee,"P",{"data-svelte-h":!0}),m(le)!=="svelte-sl2427"&&(le.innerHTML=tl),Ye=n(Ee),r(z.$$.fragment,Ee),Ee.forEach(t),Xe=n(w),F=o(w,"LI",{});var Ae=ae(F);te=o(Ae,"P",{"data-svelte-h":!0}),m(te)!=="svelte-1nvuilw"&&(te.innerHTML=sl),Re=n(Ae),r(H.$$.fragment,Ae),Ae.forEach(t),Ne=n(w),S=o(w,"LI",{});var Ve=ae(S);se=o(Ve,"P",{"data-svelte-h":!0}),m(se)!=="svelte-m52y4l"&&(se.innerHTML=al),Le=n(Ve),r(q.$$.fragment,Ve),Ve.forEach(t),w.forEach(t),Ge=n(e),P=o(e,"P",{"data-svelte-h":!0}),m(P)!=="svelte-t0b8b6"&&(P.innerHTML=nl),We=n(e),r(D.$$.fragment,e),Be=n(e),K=o(e,"OL",{"data-svelte-h":!0}),m(K)!=="svelte-1auebwz"&&(K.innerHTML=il),_e=n(e),r(O.$$.fragment,e),ke=n(e),ie=o(e,"P",{}),ae(ie).forEach(t),this.h()},h(){Qe(p,"name","hf:doc:metadata"),Qe(p,"content",jl),Qe(f,"start","2")},m(e,l){y(document.head,p),s(e,b,l),s(e,j,l),s(e,ee,l),M(U,e,l),s(e,pe,l),s(e,Z,l),s(e,me,l),M(C,e,l),s(e,ce,l),s(e,v,l),s(e,re,l),s(e,I,l),s(e,Me,l),M($,e,l),s(e,de,l),s(e,f,l),s(e,he,l),M(J,e,l),s(e,ue,l),M(G,e,l),s(e,ye,l),s(e,W,l),s(e,je,l),M(B,e,l),s(e,Te,l),M(_,e,l),s(e,we,l),s(e,k,l),s(e,fe,l),s(e,x,l),s(e,Je,l),M(E,e,l),s(e,ge,l),M(A,e,l),s(e,be,l),M(V,e,l),s(e,Ue,l),s(e,Y,l),s(e,Ze,l),s(e,X,l),s(e,Ce,l),s(e,R,l),s(e,ve,l),M(N,e,l),s(e,Ie,l),s(e,L,l),s(e,$e,l),s(e,T,l),y(T,Q),y(Q,le),y(Q,Ye),M(z,Q,null),y(T,Xe),y(T,F),y(F,te),y(F,Re),M(H,F,null),y(T,Ne),y(T,S),y(S,se),y(S,Le),M(q,S,null),s(e,Ge,l),s(e,P,l),s(e,We,l),M(D,e,l),s(e,Be,l),s(e,K,l),s(e,_e,l),M(O,e,l),s(e,ke,l),s(e,ie,l),xe=!0},p(e,[l]){const w={};l&2&&(w.$$scope={dirty:l,ctx:e}),J.$set(w)},i(e){xe||(d(U.$$.fragment,e),d(C.$$.fragment,e),d($.$$.fragment,e),d(J.$$.fragment,e),d(G.$$.fragment,e),d(B.$$.fragment,e),d(_.$$.fragment,e),d(E.$$.fragment,e),d(A.$$.fragment,e),d(V.$$.fragment,e),d(N.$$.fragment,e),d(z.$$.fragment,e),d(H.$$.fragment,e),d(q.$$.fragment,e),d(D.$$.fragment,e),d(O.$$.fragment,e),xe=!0)},o(e){h(U.$$.fragment,e),h(C.$$.fragment,e),h($.$$.fragment,e),h(J.$$.fragment,e),h(G.$$.fragment,e),h(B.$$.fragment,e),h(_.$$.fragment,e),h(E.$$.fragment,e),h(A.$$.fragment,e),h(V.$$.fragment,e),h(N.$$.fragment,e),h(z.$$.fragment,e),h(H.$$.fragment,e),h(q.$$.fragment,e),h(D.$$.fragment,e),h(O.$$.fragment,e),xe=!1},d(e){e&&(t(b),t(j),t(ee),t(pe),t(Z),t(me),t(ce),t(v),t(re),t(I),t(Me),t(de),t(f),t(he),t(ue),t(ye),t(W),t(je),t(Te),t(we),t(k),t(fe),t(x),t(Je),t(ge),t(be),t(Ue),t(Y),t(Ze),t(X),t(Ce),t(R),t(ve),t(Ie),t(L),t($e),t(T),t(Ge),t(P),t(We),t(Be),t(K),t(_e),t(ke),t(ie)),t(p),u(U,e),u(C,e),u($,e),u(J,e),u(G,e),u(B,e),u(_,e),u(E,e),u(A,e),u(V,e),u(N,e),u(z),u(H),u(q),u(D,e),u(O,e)}}}const jl='{"title":"Custom Model","local":"custom-model","sections":[{"title":"Creating a Custom Model","local":"creating-a-custom-model","sections":[],"depth":2},{"title":"Running the Evaluation","local":"running-the-evaluation","sections":[{"title":"Using the Command Line","local":"using-the-command-line","sections":[],"depth":3},{"title":"Using the Python API","local":"using-the-python-api","sections":[],"depth":3}],"depth":2},{"title":"Required Methods","local":"required-methods","sections":[],"depth":2},{"title":"Enabling Caching (Recommended)","local":"enabling-caching-recommended","sections":[],"depth":2},{"title":"Best Practices","local":"best-practices","sections":[],"depth":2}],"depth":1}';function Tl(oe){return pl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ul extends cl{constructor(p){super(),rl(this,p,Tl,yl,ol,{})}}export{Ul as component};