lighteval/pr_1037/en/_app/immutable/nodes/7.eebdac89.js

import{s as kl,n as xl,o as Rl}from"../chunks/scheduler.3a17fb72.js";import{S as El,i as _l,e as r,s as n,c as i,q as Nl,h as Ql,a as d,d as t,b as a,f as ml,g as p,j as y,r as Al,k as Ml,l as dl,m as s,n as o,t as m,o as M,p as c}from"../chunks/index.093f8863.js";import{C as Sl,H as h,E as Yl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.2631eeef.js";import{C as T}from"../chunks/CodeBlock.b3d2386f.js";function Xl(yl){let u,ye,re,he,w,ue,U,Te,g,hl=`Lighteval allows you to evaluate custom model implementations by creating a custom model class that inherits from <code>LightevalModel</code>.
This is useful when you want to evaluate models that aren’t directly supported by the standard backends and providers (Transformers, VLLM, etc.), or
if you want to add your own pre/post-processing logic.`,Je,f,we,j,Ue,b,ul="Create a Python file containing your custom model implementation. The model must inherit from <code>LightevalModel</code> and implement all required methods.",ge,C,Tl="Here’s a basic example:",fe,$,je,I,be,G,Jl="The custom model file should contain exactly one class that inherits from <code>LightevalModel</code>. This class will be automatically detected and instantiated when loading the model.",Ce,J,wl="<p>You can find a complete example of a custom model implementation in <code>examples/custom_models/google_translate_model.py</code>.</p>",$e,Z,Ie,B,Ul="You can evaluate your custom model using either the command-line interface or the Python API.",Ge,v,Ze,W,Be,V,gl="The command takes three required arguments:",ve,k,fl="<li><strong>Model name</strong>: Used for tracking in results/logs</li> <li><strong>Model implementation file path</strong>: Path to your Python file containing the custom model</li> <li><strong>Tasks</strong>: Tasks to evaluate on (same format as other backends)</li>",We,x,Ve,R,ke,E,xe,_,jl="Your custom model must implement these core methods:",Re,N,Ee,Q,bl="For generating text until a stop sequence or max tokens is reached. This is used for generative evaluations.",_e,A,Ne,S,Qe,Y,Cl="For computing log probabilities of specific continuations. This is used for multiple choice logprob evaluations.",Ae,X,Se,z,Ye,F,$l="For computing rolling log probabilities of sequences. This is used for perplexity metrics.",Xe,L,ze,H,Il="See the <code>LightevalModel</code> base class documentation for detailed method signatures and requirements.",Fe,q,Le,P,Gl=`Lighteval includes a caching system that can significantly speed up evaluations by storing and reusing model predictions.
To enable caching in your custom model:`,He,D,qe,K,Pe,O,De,ee,Ke,le,ce,cl,te,Oe,se,Zl='For detailed information about the caching system, see the <a href="caching">Caching Documentation</a>.',el,ne,ll,ae,tl,ie,Bl="<li><strong>Import Errors</strong>: Ensure all required dependencies are installed</li> <li><strong>Method Signature Errors</strong>: Verify your methods match the expected signatures</li> <li><strong>Caching Issues</strong>: Check that cache decorators are applied correctly</li> <li><strong>Performance Issues</strong>: Consider implementing batching and caching</li>",sl,pe,nl,oe,vl="<li>Use the <code>--max-samples</code> flag to test with a small dataset</li> <li>Enable detailed logging to see what’s happening</li> <li>Test individual methods in isolation</li> <li>Check the example implementations for reference</li>",al,me,Wl='For more detailed information about custom model implementation, see the <a href="package_reference/models">Model Reference</a>.',il,Me,pl,de,ol;return w=new Sl({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new h({props:{title:"Evaluating Custom Models",local:"evaluating-custom-models",headingTag:"h1"}}),f=new h({props:{title:"Creating a Custom Model",local:"creating-a-custom-model",headingTag:"h2"}}),j=new h({props:{title:"Step 1: Create Your Model Implementation",local:"step-1-create-your-model-implementation",headingTag:"h3"}}),$=new T({props:{code:"ZnJvbSUyMGxpZ2h0ZXZhbC5tb2RlbHMuYWJzdHJhY3RfbW9kZWwlMjBpbXBvcnQlMjBMaWdodGV2YWxNb2RlbCUwQWZyb20lMjBsaWdodGV2YWwubW9kZWxzLm1vZGVsX291dHB1dCUyMGltcG9ydCUyME1vZGVsUmVzcG9uc2UlMEFmcm9tJTIwbGlnaHRldmFsLnRhc2tzLnJlcXVlc3RzJTIwaW1wb3J0JTIwRG9jJTJDJTIwU2FtcGxpbmdNZXRob2QlMEFmcm9tJTIwbGlnaHRldmFsLnV0aWxzLmNhY2hlX21hbmFnZW1lbnQlMjBpbXBvcnQlMjBTYW1wbGVDYWNoZSUyQyUyMGNhY2hlZCUwQSUwQWNsYXNzJTIwTXlDdXN0b21Nb2RlbChMaWdodGV2YWxNb2RlbCklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmJTJDJTIwY29uZmlnKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN1cGVyKCkuX19pbml0X18oY29uZmlnKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEluaXRpYWxpemUlMjB5b3VyJTIwbW9kZWwlMjBoZXJlLi4uJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwRW5hYmxlJTIwY2FjaGluZyUyMChyZWNvbW1lbmRlZCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLl9jYWNoZSUyMCUzRCUyMFNhbXBsZUNhY2hlKGNvbmZpZyklMEElMEElMjAlMjAlMjAlMjAlNDBjYWNoZWQoU2FtcGxpbmdNZXRob2QuR0VORVJBVElWRSklMEElMjAlMjAlMjAlMjBkZWYlMjBncmVlZHlfdW50aWwoc2VsZiUyQyUyMGRvY3MlM0ElMjBMaXN0JTVCRG9jJTVEKSUyMC0lM0UlMjBMaXN0JTVCTW9kZWxSZXNwb25zZSU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEltcGxlbWVudCUyMGdlbmVyYXRpb24lMjBsb2dpYyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhc3MlMEElMEElMjAlMjAlMjAlMjAlNDBjYWNoZWQoU2FtcGxpbmdNZXRob2QuTE9HUFJPQlMpJTBBJTIwJTIwJTIwJTIwZGVmJTIwbG9nbGlrZWxpaG9vZChzZWxmJTJDJTIwZG9jcyUzQSUyMExpc3QlNUJEb2MlNUQpJTIwLSUzRSUyMExpc3QlNUJNb2RlbFJlc3BvbnNlJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwSW1wbGVtZW50JTIwbG9nbGlrZWxpaG9vZCUyMGNvbXB1dGF0aW9uJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGFzcyUwQSUwQSUyMCUyMCUyMCUyMCU0MGNhY2hlZChTYW1wbGluZ01ldGhvZC5QRVJQTEVYSVRZKSUwQSUyMCUyMCUyMCUyMGRlZiUyMGxvZ2xpa2VsaWhvb2Rfcm9sbGluZyhzZWxmJTJDJTIwZG9jcyUzQSUyMExpc3QlNUJEb2MlNUQpJTIwLSUzRSUyMExpc3QlNUJNb2RlbFJlc3BvbnNlJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwSW1wbGVtZW50JTIwcm9sbGluZyUyMGxvZ2xpa2VsaWhvb2QlMjBjb21wdXRhdGlvbiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhc3M=",highlighted:`<span class="hljs-keyword">from</span> lighteval.models.abstract_model <span class="hljs-keyword">import</span> LightevalModel
<span class="hljs-keyword">from</span> lighteval.models.model_output <span class="hljs-keyword">import</span> ModelResponse
<span class="hljs-keyword">from</span> lighteval.tasks.requests <span class="hljs-keyword">import</span> Doc, SamplingMethod
<span class="hljs-keyword">from</span> lighteval.utils.cache_management <span class="hljs-keyword">import</span> SampleCache, cached

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyCustomModel</span>(<span class="hljs-title class_ inherited__">LightevalModel</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
        <span class="hljs-built_in">super</span>().__init__(config)
        <span class="hljs-comment"># Initialize your model here...</span>

        <span class="hljs-comment"># Enable caching (recommended)</span>
        self._cache = SampleCache(config)

<span class="hljs-meta">    @cached(<span class="hljs-params">SamplingMethod.GENERATIVE</span>)</span>
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">greedy_until</span>(<span class="hljs-params">self, docs: <span class="hljs-type">List</span>[Doc]</span>) -&gt; <span class="hljs-type">List</span>[ModelResponse]:
        <span class="hljs-comment"># Implement generation logic</span>
        <span class="hljs-keyword">pass</span>

<span class="hljs-meta">    @cached(<span class="hljs-params">SamplingMethod.LOGPROBS</span>)</span>
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">loglikelihood</span>(<span class="hljs-params">self, docs: <span class="hljs-type">List</span>[Doc]</span>) -&gt; <span class="hljs-type">List</span>[ModelResponse]:
        <span class="hljs-comment"># Implement loglikelihood computation</span>
        <span class="hljs-keyword">pass</span>

<span class="hljs-meta">    @cached(<span class="hljs-params">SamplingMethod.PERPLEXITY</span>)</span>
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">loglikelihood_rolling</span>(<span class="hljs-params">self, docs: <span class="hljs-type">List</span>[Doc]</span>) -&gt; <span class="hljs-type">List</span>[ModelResponse]:
        <span class="hljs-comment"># Implement rolling loglikelihood computation</span>
        <span class="hljs-keyword">pass</span>`,wrap:!1}}),I=new h({props:{title:"Step 2: Model File Requirements",local:"step-2-model-file-requirements",headingTag:"h3"}}),Z=new h({props:{title:"Running the Evaluation",local:"running-the-evaluation",headingTag:"h2"}}),v=new h({props:{title:"Using the Command Line",local:"using-the-command-line",headingTag:"h3"}}),W=new T({props:{code:"bGlnaHRldmFsJTIwY3VzdG9tJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIyZ29vZ2xlLXRyYW5zbGF0ZSUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMmV4YW1wbGVzJTJGY3VzdG9tX21vZGVscyUyRmdvb2dsZV90cmFuc2xhdGVfbW9kZWwucHklMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAlMjJsaWdodGV2YWwlN0N3bXQyMCUzQWZyLWRlJTdDMCUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbWF4LXNhbXBsZXMlMjAxMA==",highlighted:`lighteval custom \\
    <span class="hljs-string">&quot;google-translate&quot;</span> \\
    <span class="hljs-string">&quot;examples/custom_models/google_translate_model.py&quot;</span> \\
    <span class="hljs-string">&quot;lighteval|wmt20:fr-de|0&quot;</span> \\
    --max-samples 10`,wrap:!1}}),x=new h({props:{title:"Using the Python API",local:"using-the-python-api",headingTag:"h3"}}),R=new T({props:{code:"ZnJvbSUyMGxpZ2h0ZXZhbC5sb2dnaW5nLmV2YWx1YXRpb25fdHJhY2tlciUyMGltcG9ydCUyMEV2YWx1YXRpb25UcmFja2VyJTBBZnJvbSUyMGxpZ2h0ZXZhbC5tb2RlbHMuY3VzdG9tLmN1c3RvbV9tb2RlbCUyMGltcG9ydCUyMEN1c3RvbU1vZGVsQ29uZmlnJTBBZnJvbSUyMGxpZ2h0ZXZhbC5waXBlbGluZSUyMGltcG9ydCUyMFBpcGVsaW5lJTJDJTIwUGlwZWxpbmVQYXJhbWV0ZXJzJTJDJTIwUGFyYWxsZWxpc21NYW5hZ2VyJTBBJTBBJTIzJTIwU2V0JTIwdXAlMjBldmFsdWF0aW9uJTIwdHJhY2tpbmclMEFldmFsdWF0aW9uX3RyYWNrZXIlMjAlM0QlMjBFdmFsdWF0aW9uVHJhY2tlciglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIycmVzdWx0cyUyMiUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfZGV0YWlscyUzRFRydWUlMEEpJTBBJTBBJTIzJTIwQ29uZmlndXJlJTIwdGhlJTIwcGlwZWxpbmUlMEFwaXBlbGluZV9wYXJhbXMlMjAlM0QlMjBQaXBlbGluZVBhcmFtZXRlcnMoJTBBJTIwJTIwJTIwJTIwbGF1bmNoZXJfdHlwZSUzRFBhcmFsbGVsaXNtTWFuYWdlci5DVVNUT00lMkMlMEEpJTBBJTBBJTIzJTIwQ29uZmlndXJlJTIweW91ciUyMGN1c3RvbSUyMG1vZGVsJTBBbW9kZWxfY29uZmlnJTIwJTNEJTIwQ3VzdG9tTW9kZWxDb25maWcoJTBBJTIwJTIwJTIwJTIwbW9kZWxfbmFtZSUzRCUyMm15LWN1c3RvbS1tb2RlbCUyMiUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX2RlZmluaXRpb25fZmlsZV9wYXRoJTNEJTIycGF0aCUyRnRvJTJGbXlfbW9kZWwucHklMjIlMEEpJTBBJTBBJTIzJTIwQ3JlYXRlJTIwYW5kJTIwcnVuJTIwdGhlJTIwcGlwZWxpbmUlMEFwaXBlbGluZSUyMCUzRCUyMFBpcGVsaW5lKCUwQSUyMCUyMCUyMCUyMHRhc2tzJTNEJTIybGVhZGVyYm9hcmQlN0N0cnV0aGZ1bHFhJTNBbWMlN0MwJTIyJTJDJTBBJTIwJTIwJTIwJTIwcGlwZWxpbmVfcGFyYW1ldGVycyUzRHBpcGVsaW5lX3BhcmFtcyUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fdHJhY2tlciUzRGV2YWx1YXRpb25fdHJhY2tlciUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX2NvbmZpZyUzRG1vZGVsX2NvbmZpZyUwQSklMEElMEFwaXBlbGluZS5ldmFsdWF0ZSgpJTBBcGlwZWxpbmUuc2F2ZV9hbmRfcHVzaF9yZXN1bHRzKCk=",highlighted:`<span class="hljs-keyword">from</span> lighteval.logging.evaluation_tracker <span class="hljs-keyword">import</span> EvaluationTracker
<span class="hljs-keyword">from</span> lighteval.models.custom.custom_model <span class="hljs-keyword">import</span> CustomModelConfig
<span class="hljs-keyword">from</span> lighteval.pipeline <span class="hljs-keyword">import</span> Pipeline, PipelineParameters, ParallelismManager

<span class="hljs-comment"># Set up evaluation tracking</span>
evaluation_tracker = EvaluationTracker(
    output_dir=<span class="hljs-string">&quot;results&quot;</span>,
    save_details=<span class="hljs-literal">True</span>
)

<span class="hljs-comment"># Configure the pipeline</span>
pipeline_params = PipelineParameters(
    launcher_type=ParallelismManager.CUSTOM,
)

<span class="hljs-comment"># Configure your custom model</span>
model_config = CustomModelConfig(
    model_name=<span class="hljs-string">&quot;my-custom-model&quot;</span>,
    model_definition_file_path=<span class="hljs-string">&quot;path/to/my_model.py&quot;</span>
)

<span class="hljs-comment"># Create and run the pipeline</span>
pipeline = Pipeline(
    tasks=<span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span>,
    pipeline_parameters=pipeline_params,
    evaluation_tracker=evaluation_tracker,
    model_config=model_config
)

pipeline.evaluate()
pipeline.save_and_push_results()`,wrap:!1}}),E=new h({props:{title:"Required Methods",local:"required-methods",headingTag:"h2"}}),N=new h({props:{title:"greedy_until",local:"greedyuntil",headingTag:"h3"}}),A=new T({props:{code:"ZGVmJTIwZ3JlZWR5X3VudGlsKHNlbGYlMkMlMjBkb2NzJTNBJTIwbGlzdCU1QkRvYyU1RCklMjAtJTNFJTIwbGlzdCU1Qk1vZGVsUmVzcG9uc2UlNUQlM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjBHZW5lcmF0ZSUyMHRleHQlMjB1bnRpbCUyMHN0b3AlMjBzZXF1ZW5jZSUyMG9yJTIwbWF4JTIwdG9rZW5zLiUwQSUwQSUyMCUyMCUyMCUyMEFyZ3MlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkb2NzJTNBJTIwbGlzdCUyMG9mJTIwZG9jdW1lbnRzJTIwY29udGFpbmluZyUyMHByb21wdHMlMjBhbmQlMjBnZW5lcmF0aW9uJTIwcGFyYW1ldGVycyUwQSUwQSUyMCUyMCUyMCUyMFJldHVybnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsaXN0JTIwb2YlMjBtb2RlbCUyMHJlc3BvbnNlcyUyMHdpdGglMjBnZW5lcmF0ZWQlMjB0ZXh0JTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwcGFzcw==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">greedy_until</span>(<span class="hljs-params">self, docs: <span class="hljs-built_in">list</span>[Doc]</span>) -&gt; <span class="hljs-built_in">list</span>[ModelResponse]:
    <span class="hljs-string">&quot;&quot;&quot;
    Generate text until stop sequence or max tokens.

    Args:
        docs: list of documents containing prompts and generation parameters

    Returns:
        list of model responses with generated text
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">pass</span>`,wrap:!1}}),S=new h({props:{title:"loglikelihood",local:"loglikelihood",headingTag:"h3"}}),X=new T({props:{code:"ZGVmJTIwbG9nbGlrZWxpaG9vZChzZWxmJTJDJTIwZG9jcyUzQSUyMGxpc3QlNUJEb2MlNUQpJTIwLSUzRSUyMGxpc3QlNUJNb2RlbFJlc3BvbnNlJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwQ29tcHV0ZSUyMGxvZyUyMHByb2JhYmlsaXRpZXMlMjBvZiUyMGNvbnRpbnVhdGlvbnMuJTBBJTBBJTIwJTIwJTIwJTIwQXJncyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRvY3MlM0ElMjBsaXN0JTIwb2YlMjBkb2N1bWVudHMlMjBjb250YWluaW5nJTIwY29udGV4dCUyMGFuZCUyMGNvbnRpbnVhdGlvbiUyMHBhaXJzJTBBJTBBJTIwJTIwJTIwJTIwUmV0dXJucyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxpc3QlMjBvZiUyMG1vZGVsJTIwcmVzcG9uc2VzJTIwd2l0aCUyMGxvZyUyMHByb2JhYmlsaXRpZXMlMEElMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjBwYXNz",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">loglikelihood</span>(<span class="hljs-params">self, docs: <span class="hljs-built_in">list</span>[Doc]</span>) -&gt; <span class="hljs-built_in">list</span>[ModelResponse]:
    <span class="hljs-string">&quot;&quot;&quot;
    Compute log probabilities of continuations.

    Args:
        docs: list of documents containing context and continuation pairs

    Returns:
        list of model responses with log probabilities
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">pass</span>`,wrap:!1}}),z=new h({props:{title:"loglikelihood_rolling",local:"loglikelihoodrolling",headingTag:"h3"}}),L=new T({props:{code:"ZGVmJTIwbG9nbGlrZWxpaG9vZF9yb2xsaW5nKHNlbGYlMkMlMjBkb2NzJTNBJTIwbGlzdCU1QkRvYyU1RCklMjAtJTNFJTIwbGlzdCU1Qk1vZGVsUmVzcG9uc2UlNUQlM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjBDb21wdXRlJTIwcm9sbGluZyUyMGxvZyUyMHByb2JhYmlsaXRpZXMlMjBvZiUyMHNlcXVlbmNlcy4lMEElMEElMjAlMjAlMjAlMjBBcmdzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZG9jcyUzQSUyMGxpc3QlMjBvZiUyMGRvY3VtZW50cyUyMGNvbnRhaW5pbmclMjB0ZXh0JTIwc2VxdWVuY2VzJTBBJTBBJTIwJTIwJTIwJTIwUmV0dXJucyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxpc3QlMjBvZiUyMG1vZGVsJTIwcmVzcG9uc2VzJTIwd2l0aCUyMHJvbGxpbmclMjBsb2clMjBwcm9iYWJpbGl0aWVzJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwcGFzcw==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">loglikelihood_rolling</span>(<span class="hljs-params">self, docs: <span class="hljs-built_in">list</span>[Doc]</span>) -&gt; <span class="hljs-built_in">list</span>[ModelResponse]:
    <span class="hljs-string">&quot;&quot;&quot;
    Compute rolling log probabilities of sequences.

    Args:
        docs: list of documents containing text sequences

    Returns:
        list of model responses with rolling log probabilities
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">pass</span>`,wrap:!1}}),q=new h({props:{title:"Enabling Caching (Recommended)",local:"enabling-caching-recommended",headingTag:"h2"}}),D=new h({props:{title:"Step 1: Import Caching Components",local:"step-1-import-caching-components",headingTag:"h3"}}),K=new T({props:{code:"ZnJvbSUyMGxpZ2h0ZXZhbC51dGlscy5jYWNoZV9tYW5hZ2VtZW50JTIwaW1wb3J0JTIwU2FtcGxlQ2FjaGUlMkMlMjBjYWNoZWQ=",highlighted:'<span class="hljs-keyword">from</span> lighteval.utils.cache_management <span class="hljs-keyword">import</span> SampleCache, cached',wrap:!1}}),O=new h({props:{title:"Step 2: Initialize Cache in Constructor",local:"step-2-initialize-cache-in-constructor",headingTag:"h3"}}),ee=new T({props:{code:"ZGVmJTIwX19pbml0X18oc2VsZiUyQyUyMGNvbmZpZyklM0ElMEElMjAlMjAlMjAlMjBzdXBlcigpLl9faW5pdF9fKGNvbmZpZyklMEElMjAlMjAlMjAlMjAlMjMlMjBZb3VyJTIwaW5pdGlhbGl6YXRpb24lMjBjb2RlLi4uJTBBJTIwJTIwJTIwJTIwc2VsZi5fY2FjaGUlMjAlM0QlMjBTYW1wbGVDYWNoZShjb25maWcp",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
    <span class="hljs-built_in">super</span>().__init__(config)
    <span class="hljs-comment"># Your initialization code...</span>
    self._cache = SampleCache(config)`,wrap:!1}}),te=new T({props:{code:"JTQwY2FjaGVkKFNhbXBsaW5nTWV0aG9kLkdFTkVSQVRJVkUpJTBBZGVmJTIwZ3JlZWR5X3VudGlsKHNlbGYlMkMlMjBkb2NzJTNBJTIwTGlzdCU1QkRvYyU1RCklMjAtJTNFJTIwTGlzdCU1Qk1vZGVsUmVzcG9uc2UlNUQlM0ElMEElMjAlMjAlMjAlMjAlMjMlMjBZb3VyJTIwaW1wbGVtZW50YXRpb24uLi4=",highlighted:`<span class="hljs-meta">@cached(<span class="hljs-params">SamplingMethod.GENERATIVE</span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">greedy_until</span>(<span class="hljs-params">self, docs: <span class="hljs-type">List</span>[Doc]</span>) -&gt; <span class="hljs-type">List</span>[ModelResponse]:
    <span class="hljs-comment"># Your implementation...</span>`,wrap:!1}}),ne=new h({props:{title:"Troubleshooting",local:"troubleshooting",headingTag:"h2"}}),ae=new h({props:{title:"Common Issues",local:"common-issues",headingTag:"h3"}}),pe=new h({props:{title:"Debugging Tips",local:"debugging-tips",headingTag:"h3"}}),Me=new Yl({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/evaluating-a-custom-model.mdx"}}),{c(){u=r("meta"),ye=n(),re=r("p"),he=n(),i(w.$$.fragment),ue=n(),i(U.$$.fragment),Te=n(),g=r("p"),g.innerHTML=hl,Je=n(),i(f.$$.fragment),we=n(),i(j.$$.fragment),Ue=n(),b=r("p"),b.innerHTML=ul,ge=n(),C=r("p"),C.textContent=Tl,fe=n(),i($.$$.fragment),je=n(),i(I.$$.fragment),be=n(),G=r("p"),G.innerHTML=Jl,Ce=n(),J=r("blockquote"),J.innerHTML=wl,$e=n(),i(Z.$$.fragment),Ie=n(),B=r("p"),B.textContent=Ul,Ge=n(),i(v.$$.fragment),Ze=n(),i(W.$$.fragment),Be=n(),V=r("p"),V.textContent=gl,ve=n(),k=r("ul"),k.innerHTML=fl,We=n(),i(x.$$.fragment),Ve=n(),i(R.$$.fragment),ke=n(),i(E.$$.fragment),xe=n(),_=r("p"),_.textContent=jl,Re=n(),i(N.$$.fragment),Ee=n(),Q=r("p"),Q.textContent=bl,_e=n(),i(A.$$.fragment),Ne=n(),i(S.$$.fragment),Qe=n(),Y=r("p"),Y.textContent=Cl,Ae=n(),i(X.$$.fragment),Se=n(),i(z.$$.fragment),Ye=n(),F=r("p"),F.textContent=$l,Xe=n(),i(L.$$.fragment),ze=n(),H=r("p"),H.innerHTML=Il,Fe=n(),i(q.$$.fragment),Le=n(),P=r("p"),P.textContent=Gl,He=n(),i(D.$$.fragment),qe=n(),i(K.$$.fragment),Pe=n(),i(O.$$.fragment),De=n(),i(ee.$$.fragment),Ke=n(),le=r("ol"),ce=r("li"),cl=Nl(`Add cache decorators to your prediction methods:

	`),i(te.$$.fragment),Oe=n(),se=r("p"),se.innerHTML=Zl,el=n(),i(ne.$$.fragment),ll=n(),i(ae.$$.fragment),tl=n(),ie=r("ol"),ie.innerHTML=Bl,sl=n(),i(pe.$$.fragment),nl=n(),oe=r("ul"),oe.innerHTML=vl,al=n(),me=r("p"),me.innerHTML=Wl,il=n(),i(Me.$$.fragment),pl=n(),de=r("p"),this.h()},l(e){const l=Ql("svelte-u9bgzb",document.head);u=d(l,"META",{name:!0,content:!0}),l.forEach(t),ye=a(e),re=d(e,"P",{}),ml(re).forEach(t),he=a(e),p(w.$$.fragment,e),ue=a(e),p(U.$$.fragment,e),Te=a(e),g=d(e,"P",{"data-svelte-h":!0}),y(g)!=="svelte-15yahec"&&(g.innerHTML=hl),Je=a(e),p(f.$$.fragment,e),we=a(e),p(j.$$.fragment,e),Ue=a(e),b=d(e,"P",{"data-svelte-h":!0}),y(b)!=="svelte-64btiy"&&(b.innerHTML=ul),ge=a(e),C=d(e,"P",{"data-svelte-h":!0}),y(C)!=="svelte-1frc3d"&&(C.textContent=Tl),fe=a(e),p($.$$.fragment,e),je=a(e),p(I.$$.fragment,e),be=a(e),G=d(e,"P",{"data-svelte-h":!0}),y(G)!=="svelte-4gi028"&&(G.innerHTML=Jl),Ce=a(e),J=d(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),y(J)!=="svelte-1m7vrvf"&&(J.innerHTML=wl),$e=a(e),p(Z.$$.fragment,e),Ie=a(e),B=d(e,"P",{"data-svelte-h":!0}),y(B)!=="svelte-md1ogy"&&(B.textContent=Ul),Ge=a(e),p(v.$$.fragment,e),Ze=a(e),p(W.$$.fragment,e),Be=a(e),V=d(e,"P",{"data-svelte-h":!0}),y(V)!=="svelte-9ha93v"&&(V.textContent=gl),ve=a(e),k=d(e,"UL",{"data-svelte-h":!0}),y(k)!=="svelte-7oyst4"&&(k.innerHTML=fl),We=a(e),p(x.$$.fragment,e),Ve=a(e),p(R.$$.fragment,e),ke=a(e),p(E.$$.fragment,e),xe=a(e),_=d(e,"P",{"data-svelte-h":!0}),y(_)!=="svelte-abp4ax"&&(_.textContent=jl),Re=a(e),p(N.$$.fragment,e),Ee=a(e),Q=d(e,"P",{"data-svelte-h":!0}),y(Q)!=="svelte-1nk29lo"&&(Q.textContent=bl),_e=a(e),p(A.$$.fragment,e),Ne=a(e),p(S.$$.fragment,e),Qe=a(e),Y=d(e,"P",{"data-svelte-h":!0}),y(Y)!=="svelte-1bufz8k"&&(Y.textContent=Cl),Ae=a(e),p(X.$$.fragment,e),Se=a(e),p(z.$$.fragment,e),Ye=a(e),F=d(e,"P",{"data-svelte-h":!0}),y(F)!=="svelte-dqj7yp"&&(F.textContent=$l),Xe=a(e),p(L.$$.fragment,e),ze=a(e),H=d(e,"P",{"data-svelte-h":!0}),y(H)!=="svelte-845vib"&&(H.innerHTML=Il),Fe=a(e),p(q.$$.fragment,e),Le=a(e),P=d(e,"P",{"data-svelte-h":!0}),y(P)!=="svelte-1eshig5"&&(P.textContent=Gl),He=a(e),p(D.$$.fragment,e),qe=a(e),p(K.$$.fragment,e),Pe=a(e),p(O.$$.fragment,e),De=a(e),p(ee.$$.fragment,e),Ke=a(e),le=d(e,"OL",{start:!0});var Vl=ml(le);ce=d(Vl,"LI",{});var rl=ml(ce);cl=Al(rl,`Add cache decorators to your prediction methods:

	`),p(te.$$.fragment,rl),rl.forEach(t),Vl.forEach(t),Oe=a(e),se=d(e,"P",{"data-svelte-h":!0}),y(se)!=="svelte-eqax9m"&&(se.innerHTML=Zl),el=a(e),p(ne.$$.fragment,e),ll=a(e),p(ae.$$.fragment,e),tl=a(e),ie=d(e,"OL",{"data-svelte-h":!0}),y(ie)!=="svelte-hifd4u"&&(ie.innerHTML=Bl),sl=a(e),p(pe.$$.fragment,e),nl=a(e),oe=d(e,"UL",{"data-svelte-h":!0}),y(oe)!=="svelte-1g8hpki"&&(oe.innerHTML=vl),al=a(e),me=d(e,"P",{"data-svelte-h":!0}),y(me)!=="svelte-15nhhnt"&&(me.innerHTML=Wl),il=a(e),p(Me.$$.fragment,e),pl=a(e),de=d(e,"P",{}),ml(de).forEach(t),this.h()},h(){Ml(u,"name","hf:doc:metadata"),Ml(u,"content",zl),Ml(J,"class","tip"),Ml(le,"start","3")},m(e,l){dl(document.head,u),s(e,ye,l),s(e,re,l),s(e,he,l),o(w,e,l),s(e,ue,l),o(U,e,l),s(e,Te,l),s(e,g,l),s(e,Je,l),o(f,e,l),s(e,we,l),o(j,e,l),s(e,Ue,l),s(e,b,l),s(e,ge,l),s(e,C,l),s(e,fe,l),o($,e,l),s(e,je,l),o(I,e,l),s(e,be,l),s(e,G,l),s(e,Ce,l),s(e,J,l),s(e,$e,l),o(Z,e,l),s(e,Ie,l),s(e,B,l),s(e,Ge,l),o(v,e,l),s(e,Ze,l),o(W,e,l),s(e,Be,l),s(e,V,l),s(e,ve,l),s(e,k,l),s(e,We,l),o(x,e,l),s(e,Ve,l),o(R,e,l),s(e,ke,l),o(E,e,l),s(e,xe,l),s(e,_,l),s(e,Re,l),o(N,e,l),s(e,Ee,l),s(e,Q,l),s(e,_e,l),o(A,e,l),s(e,Ne,l),o(S,e,l),s(e,Qe,l),s(e,Y,l),s(e,Ae,l),o(X,e,l),s(e,Se,l),o(z,e,l),s(e,Ye,l),s(e,F,l),s(e,Xe,l),o(L,e,l),s(e,ze,l),s(e,H,l),s(e,Fe,l),o(q,e,l),s(e,Le,l),s(e,P,l),s(e,He,l),o(D,e,l),s(e,qe,l),o(K,e,l),s(e,Pe,l),o(O,e,l),s(e,De,l),o(ee,e,l),s(e,Ke,l),s(e,le,l),dl(le,ce),dl(ce,cl),o(te,ce,null),s(e,Oe,l),s(e,se,l),s(e,el,l),o(ne,e,l),s(e,ll,l),o(ae,e,l),s(e,tl,l),s(e,ie,l),s(e,sl,l),o(pe,e,l),s(e,nl,l),s(e,oe,l),s(e,al,l),s(e,me,l),s(e,il,l),o(Me,e,l),s(e,pl,l),s(e,de,l),ol=!0},p:xl,i(e){ol||(m(w.$$.fragment,e),m(U.$$.fragment,e),m(f.$$.fragment,e),m(j.$$.fragment,e),m($.$$.fragment,e),m(I.$$.fragment,e),m(Z.$$.fragment,e),m(v.$$.fragment,e),m(W.$$.fragment,e),m(x.$$.fragment,e),m(R.$$.fragment,e),m(E.$$.fragment,e),m(N.$$.fragment,e),m(A.$$.fragment,e),m(S.$$.fragment,e),m(X.$$.fragment,e),m(z.$$.fragment,e),m(L.$$.fragment,e),m(q.$$.fragment,e),m(D.$$.fragment,e),m(K.$$.fragment,e),m(O.$$.fragment,e),m(ee.$$.fragment,e),m(te.$$.fragment,e),m(ne.$$.fragment,e),m(ae.$$.fragment,e),m(pe.$$.fragment,e),m(Me.$$.fragment,e),ol=!0)},o(e){M(w.$$.fragment,e),M(U.$$.fragment,e),M(f.$$.fragment,e),M(j.$$.fragment,e),M($.$$.fragment,e),M(I.$$.fragment,e),M(Z.$$.fragment,e),M(v.$$.fragment,e),M(W.$$.fragment,e),M(x.$$.fragment,e),M(R.$$.fragment,e),M(E.$$.fragment,e),M(N.$$.fragment,e),M(A.$$.fragment,e),M(S.$$.fragment,e),M(X.$$.fragment,e),M(z.$$.fragment,e),M(L.$$.fragment,e),M(q.$$.fragment,e),M(D.$$.fragment,e),M(K.$$.fragment,e),M(O.$$.fragment,e),M(ee.$$.fragment,e),M(te.$$.fragment,e),M(ne.$$.fragment,e),M(ae.$$.fragment,e),M(pe.$$.fragment,e),M(Me.$$.fragment,e),ol=!1},d(e){e&&(t(ye),t(re),t(he),t(ue),t(Te),t(g),t(Je),t(we),t(Ue),t(b),t(ge),t(C),t(fe),t(je),t(be),t(G),t(Ce),t(J),t($e),t(Ie),t(B),t(Ge),t(Ze),t(Be),t(V),t(ve),t(k),t(We),t(Ve),t(ke),t(xe),t(_),t(Re),t(Ee),t(Q),t(_e),t(Ne),t(Qe),t(Y),t(Ae),t(Se),t(Ye),t(F),t(Xe),t(ze),t(H),t(Fe),t(Le),t(P),t(He),t(qe),t(Pe),t(De),t(Ke),t(le),t(Oe),t(se),t(el),t(ll),t(tl),t(ie),t(sl),t(nl),t(oe),t(al),t(me),t(il),t(pl),t(de)),t(u),c(w,e),c(U,e),c(f,e),c(j,e),c($,e),c(I,e),c(Z,e),c(v,e),c(W,e),c(x,e),c(R,e),c(E,e),c(N,e),c(A,e),c(S,e),c(X,e),c(z,e),c(L,e),c(q,e),c(D,e),c(K,e),c(O,e),c(ee,e),c(te),c(ne,e),c(ae,e),c(pe,e),c(Me,e)}}}const zl='{"title":"Evaluating Custom Models","local":"evaluating-custom-models","sections":[{"title":"Creating a Custom Model","local":"creating-a-custom-model","sections":[{"title":"Step 1: Create Your Model Implementation","local":"step-1-create-your-model-implementation","sections":[],"depth":3},{"title":"Step 2: Model File Requirements","local":"step-2-model-file-requirements","sections":[],"depth":3}],"depth":2},{"title":"Running the Evaluation","local":"running-the-evaluation","sections":[{"title":"Using the Command Line","local":"using-the-command-line","sections":[],"depth":3},{"title":"Using the Python API","local":"using-the-python-api","sections":[],"depth":3}],"depth":2},{"title":"Required Methods","local":"required-methods","sections":[{"title":"greedy_until","local":"greedyuntil","sections":[],"depth":3},{"title":"loglikelihood","local":"loglikelihood","sections":[],"depth":3},{"title":"loglikelihood_rolling","local":"loglikelihoodrolling","sections":[],"depth":3}],"depth":2},{"title":"Enabling Caching (Recommended)","local":"enabling-caching-recommended","sections":[{"title":"Step 1: Import Caching Components","local":"step-1-import-caching-components","sections":[],"depth":3},{"title":"Step 2: Initialize Cache in Constructor","local":"step-2-initialize-cache-in-constructor","sections":[],"depth":3}],"depth":2},{"title":"Troubleshooting","local":"troubleshooting","sections":[{"title":"Common Issues","local":"common-issues","sections":[],"depth":3},{"title":"Debugging Tips","local":"debugging-tips","sections":[],"depth":3}],"depth":2}],"depth":1}';function Fl(yl){return Rl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Dl extends El{constructor(u){super(),_l(this,u,Fl,Xl,kl,{})}}export{Dl as component};