lighteval/pr_1264/en/_app/immutable/nodes/12.14bbb42e.js

import{s as S,n as A,o as Y}from"../chunks/scheduler.3a17fb72.js";import{S as z,i as L,e as h,s as e,c as v,h as V,a as d,d as s,b as n,f as H,g as k,j as N,k as Q,l as P,m as l,n as G,t as q,o as $,p as E}from"../chunks/index.093f8863.js";import{C as D,H as O,E as K}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.1a6c7b74.js";import{C as tt}from"../chunks/CodeBlock.321b9d9c.js";function at(X){let o,T,f,w,i,U,p,j,c,R=`If you are prototyping a task based on files that are not yet hosted on the
Hub, you can take advantage of the <code>hf_data_files</code> argument to point lighteval
at local JSON/CSV resources. This makes it easy to evaluate datasets that live
in your repo or that are generated on the fly.`,b,r,W='Internally, <code>hf_data_files</code> is passed directly to the <code>data_files</code> parameter of <code>datasets.load_dataset</code> (<a href="(https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset)">docs</a>).',I,M,x='See <a href="adding-a-custom-task">adding a custom task</a> for more information on how to create a custom task.',_,m,g,y,F=`Once the config is registered in <code>TASKS_TABLE</code>, running the task with
<code>--custom-tasks path/to/your_file.py</code> will automatically load the local data
files. You can also pass a dictionary to <code>hf_data_files</code> (e.g.
<code>{&quot;train&quot;: &quot;train.jsonl&quot;, &quot;validation&quot;: &quot;val.jsonl&quot;}</code>) to expose multiple
splits.`,Z,u,B,J,C;return i=new D({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),p=new O({props:{title:"Offline evaluation using local data files",local:"offline-evaluation-using-local-data-files",headingTag:"h1"}}),m=new tt({props:{code:"ZnJvbSUyMHBhdGhsaWIlMjBpbXBvcnQlMjBQYXRoJTBBJTBBZnJvbSUyMGxpZ2h0ZXZhbC5tZXRyaWNzJTIwaW1wb3J0JTIwTWV0cmljcyUwQWZyb20lMjBsaWdodGV2YWwudGFza3MubGlnaHRldmFsX3Rhc2slMjBpbXBvcnQlMjBMaWdodGV2YWxUYXNrQ29uZmlnJTBBZnJvbSUyMGxpZ2h0ZXZhbC50YXNrcy5yZXF1ZXN0cyUyMGltcG9ydCUyMERvYyUwQSUwQSUwQWRlZiUyMGxvY2FsX3Byb21wdChsaW5lJTNBJTIwZGljdCUyQyUyMHRhc2tfbmFtZSUzQSUyMHN0ciklMjAtJTNFJTIwRG9jJTNBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwRG9jKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRhc2tfbmFtZSUzRHRhc2tfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXJ5JTNEbGluZSU1QiUyMnF1ZXN0aW9uJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2hvaWNlcyUzRGxpbmUlNUIlMjJjaG9pY2VzJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRGxpbmUlNUIlMjJhbnN3ZXIlMjIlNUQlMEElMjAlMjAlMjAlMjApJTBBJTBBJTBBbG9jYWxfZGF0YSUyMCUzRCUyMFBhdGgoX19maWxlX18pLnBhcmVudCUyMCUyRiUyMCUyMnNhbXBsZXMlMjIlMjAlMkYlMjAlMjJmYXEuanNvbmwlMjIlMEElMEFsb2NhbF90YXNrJTIwJTNEJTIwTGlnaHRldmFsVGFza0NvbmZpZyglMEElMjAlMjAlMjAlMjBuYW1lJTNEJTIyZmFxX2V2YWwlMjIlMkMlMEElMjAlMjAlMjAlMjBwcm9tcHRfZnVuY3Rpb24lM0Rsb2NhbF9wcm9tcHQlMkMlMEElMjAlMjAlMjAlMjBoZl9yZXBvJTNEJTIyanNvbiUyMiUyQyUyMCUyMCUyMyUyMEJ1aWx0LWluJTIwc3RyZWFtaW5nJTIwbG9hZGVyJTIwZm9yJTIwanNvbiUyRmpzb25sJTIwZmlsZXMlMEElMjAlMjAlMjAlMjBoZl9zdWJzZXQlM0QlMjJkZWZhdWx0JTIyJTJDJTBBJTIwJTIwJTIwJTIwaGZfZGF0YV9maWxlcyUzRHN0cihsb2NhbF9kYXRhKSUyQyUyMCUyMCUyMyUyMENhbiUyMGFsc28lMjBiZSUyMGElMjBkaWN0JTIwbWFwcGluZyUyMHNwbGl0JTIwbmFtZXMlMjB0byUyMHBhdGhzJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbl9zcGxpdHMlM0QlNUIlMjJ0cmFpbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMG1ldHJpY3MlM0QlNUJNZXRyaWNzLkFDQ1VSQUNZJTVEJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path

<span class="hljs-keyword">from</span> lighteval.metrics <span class="hljs-keyword">import</span> Metrics
<span class="hljs-keyword">from</span> lighteval.tasks.lighteval_task <span class="hljs-keyword">import</span> LightevalTaskConfig
<span class="hljs-keyword">from</span> lighteval.tasks.requests <span class="hljs-keyword">import</span> Doc


<span class="hljs-keyword">def</span> <span class="hljs-title function_">local_prompt</span>(<span class="hljs-params">line: <span class="hljs-built_in">dict</span>, task_name: <span class="hljs-built_in">str</span></span>) -&gt; Doc:
    <span class="hljs-keyword">return</span> Doc(
        task_name=task_name,
        query=line[<span class="hljs-string">&quot;question&quot;</span>],
        choices=line[<span class="hljs-string">&quot;choices&quot;</span>],
        gold_index=line[<span class="hljs-string">&quot;answer&quot;</span>]
    )


local_data = Path(__file__).parent / <span class="hljs-string">&quot;samples&quot;</span> / <span class="hljs-string">&quot;faq.jsonl&quot;</span>

local_task = LightevalTaskConfig(
    name=<span class="hljs-string">&quot;faq_eval&quot;</span>,
    prompt_function=local_prompt,
    hf_repo=<span class="hljs-string">&quot;json&quot;</span>,  <span class="hljs-comment"># Built-in streaming loader for json/jsonl files</span>
    hf_subset=<span class="hljs-string">&quot;default&quot;</span>,
    hf_data_files=<span class="hljs-built_in">str</span>(local_data),  <span class="hljs-comment"># Can also be a dict mapping split names to paths</span>
    evaluation_splits=[<span class="hljs-string">&quot;train&quot;</span>],
    metrics=[Metrics.ACCURACY],
)`,lang:"python",wrap:!1}}),u=new K({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/offline-evaluation.md"}}),{c(){o=h("meta"),T=e(),f=h("p"),w=e(),v(i.$$.fragment),U=e(),v(p.$$.fragment),j=e(),c=h("p"),c.innerHTML=R,b=e(),r=h("p"),r.innerHTML=W,I=e(),M=h("p"),M.innerHTML=x,_=e(),v(m.$$.fragment),g=e(),y=h("p"),y.innerHTML=F,Z=e(),v(u.$$.fragment),B=e(),J=h("p"),this.h()},l(t){const a=V("svelte-u9bgzb",document.head);o=d(a,"META",{name:!0,content:!0}),a.forEach(s),T=n(t),f=d(t,"P",{}),H(f).forEach(s),w=n(t),k(i.$$.fragment,t),U=n(t),k(p.$$.fragment,t),j=n(t),c=d(t,"P",{"data-svelte-h":!0}),N(c)!=="svelte-52pw01"&&(c.innerHTML=R),b=n(t),r=d(t,"P",{"data-svelte-h":!0}),N(r)!=="svelte-15bq69p"&&(r.innerHTML=W),I=n(t),M=d(t,"P",{"data-svelte-h":!0}),N(M)!=="svelte-oqclj2"&&(M.innerHTML=x),_=n(t),k(m.$$.fragment,t),g=n(t),y=d(t,"P",{"data-svelte-h":!0}),N(y)!=="svelte-ulduhb"&&(y.innerHTML=F),Z=n(t),k(u.$$.fragment,t),B=n(t),J=d(t,"P",{}),H(J).forEach(s),this.h()},h(){Q(o,"name","hf:doc:metadata"),Q(o,"content",st)},m(t,a){P(document.head,o),l(t,T,a),l(t,f,a),l(t,w,a),G(i,t,a),l(t,U,a),G(p,t,a),l(t,j,a),l(t,c,a),l(t,b,a),l(t,r,a),l(t,I,a),l(t,M,a),l(t,_,a),G(m,t,a),l(t,g,a),l(t,y,a),l(t,Z,a),G(u,t,a),l(t,B,a),l(t,J,a),C=!0},p:A,i(t){C||(q(i.$$.fragment,t),q(p.$$.fragment,t),q(m.$$.fragment,t),q(u.$$.fragment,t),C=!0)},o(t){$(i.$$.fragment,t),$(p.$$.fragment,t),$(m.$$.fragment,t),$(u.$$.fragment,t),C=!1},d(t){t&&(s(T),s(f),s(w),s(U),s(j),s(c),s(b),s(r),s(I),s(M),s(_),s(g),s(y),s(Z),s(B),s(J)),s(o),E(i,t),E(p,t),E(m,t),E(u,t)}}}const st='{"title":"Offline evaluation using local data files","local":"offline-evaluation-using-local-data-files","sections":[],"depth":1}';function lt(X){return Y(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pt extends z{constructor(o){super(),L(this,o,lt,at,S,{})}}export{pt as component};