lighteval/pr_1221/en/_app/immutable/nodes/5.1cfd3334.js

import{s as Ie,n as be,o as Oe}from"../chunks/scheduler.3a17fb72.js";import{S as Ne,i as Qe,e as s,s as n,c,h as We,a as m,d as l,b as i,f as ke,g as o,j as f,k as _e,l as Ye,m as a,n as p,t as h,o as r,p as u}from"../chunks/index.093f8863.js";import{C as Re,H as $,E as Ze}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.5e7ea2bd.js";import{C as $e}from"../chunks/CodeBlock.09235327.js";function Ee(Te){let g,P,G,V,T,H,d,B,M,de=`Lighteval includes a caching system that can significantly speed up evaluations by storing and reusing model predictions.
This is especially useful when running the same evaluation multiple times, or comparing different evaluation metrics on the same model outputs.`,F,w,S,C,Me=`The caching system caches the predictions of the model for now (we will add tokenized input caching later).
It stores model responses objects (generations, logits, probabilities) for evaluation samples.`,D,J,z,v,we="Cached data is stored on disk using HuggingFace datasets in the following structure:",X,U,q,j,Ce="Where:",K,x,Je="<li><code>model_name</code>: The model name (path on the hub or local path)</li> <li><code>model_hash</code>: Hash of the model configuration to ensure cache invalidation when parameters change</li> <li><code>task_name</code>: Name of the evaluation task</li>",ee,y,te,A,ve="A new cache is automatically created when:",le,k,Ue="<li>Model configuration changes (different parameters, quantization, etc.)</li> <li>Model weights change (different revision, checkpoint, etc.)</li> <li>Generation parameters change (temperature, max_tokens, etc.)</li>",ae,_,je="This ensures that cached results are always consistent with your current model setup.",ne,I,ie,b,se,O,xe=`All built-in model classes in Lighteval automatically support caching. No additional configuration is needed.
For custom models you need to add a cache to the model class and decorators on all functions.`,me,N,ce,Q,oe,W,ye="To clear cache for a specific model, delete the corresponding directory:",pe,Y,he,R,Ae="To clear all caches:",re,Z,ue,E,fe,L,ge;return T=new Re({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),d=new $({props:{title:"Caching System",local:"caching-system",headingTag:"h1"}}),w=new $({props:{title:"How It Works",local:"how-it-works",headingTag:"h2"}}),J=new $({props:{title:"Cache Structure",local:"cache-structure",headingTag:"h3"}}),U=new $e({props:{code:"LmNhY2hlJTJGJTBBJUUyJTk0JTk0JUUyJTk0JTgwJUUyJTk0JTgwJTIwaHVnZ2luZ2ZhY2UlMkYlMEElMjAlMjAlMjAlMjAlRTIlOTQlOTQlRTIlOTQlODAlRTIlOTQlODAlMjBsaWdodGV2YWwlMkYlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlRTIlOTQlOTQlRTIlOTQlODAlRTIlOTQlODAlMjBwcmVkaWN0aW9ucyUyRiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCVFMiU5NCU5NCVFMiU5NCU4MCVFMiU5NCU4MCUyMCU3Qm1vZGVsX25hbWUlN0QlMkYlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlRTIlOTQlOTQlRTIlOTQlODAlRTIlOTQlODAlMjAlN0Jtb2RlbF9oYXNoJTdEJTJGJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJUUyJTk0JTk0JUUyJTk0JTgwJUUyJTk0JTgwJTIwJTdCdGFza19uYW1lJTdELnBhcnF1ZXQ=",highlighted:`<span class="language-xml">.cache/
└── huggingface/
    └── lighteval/
        └── predictions/
            └── </span><span class="hljs-template-variable">{model_name}</span><span class="language-xml">/
                └── </span><span class="hljs-template-variable">{model_hash}</span><span class="language-xml">/
                    └── </span><span class="hljs-template-variable">{task_name}</span><span class="language-xml">.parquet</span>`,wrap:!1}}),y=new $({props:{title:"Cache Recreation",local:"cache-recreation",headingTag:"h3"}}),I=new $({props:{title:"Using Caching",local:"using-caching",headingTag:"h2"}}),b=new $({props:{title:"Automatic Caching",local:"automatic-caching",headingTag:"h3"}}),N=new $({props:{title:"Cache Management",local:"cache-management",headingTag:"h2"}}),Q=new $({props:{title:"Clearing Cache",local:"clearing-cache",headingTag:"h3"}}),Y=new $e({props:{code:"cm0lMjAtcmYlMjB+JTJGLmNhY2hlJTJGaHVnZ2luZ2ZhY2UlMkZsaWdodGV2YWwlMkZwcmVkaWN0aW9ucyUyRiU3Qm1vZGVsX25hbWUlN0QlMkYlN0Jtb2RlbF9oYXNoJTdEJTJG",highlighted:'<span class="hljs-built_in">rm</span> -rf ~/.cache/huggingface/lighteval/predictions/{model_name}/{model_hash}/',wrap:!1}}),Z=new $e({props:{code:"cm0lMjAtcmYlMjB+JTJGLmNhY2hlJTJGaHVnZ2luZ2ZhY2UlMkZsaWdodGV2YWwlMkZwcmVkaWN0aW9ucw==",highlighted:'<span class="hljs-built_in">rm</span> -rf ~/.cache/huggingface/lighteval/predictions',wrap:!1}}),E=new Ze({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/caching.mdx"}}),{c(){g=s("meta"),P=n(),G=s("p"),V=n(),c(T.$$.fragment),H=n(),c(d.$$.fragment),B=n(),M=s("p"),M.textContent=de,F=n(),c(w.$$.fragment),S=n(),C=s("p"),C.textContent=Me,D=n(),c(J.$$.fragment),z=n(),v=s("p"),v.textContent=we,X=n(),c(U.$$.fragment),q=n(),j=s("p"),j.textContent=Ce,K=n(),x=s("ul"),x.innerHTML=Je,ee=n(),c(y.$$.fragment),te=n(),A=s("p"),A.textContent=ve,le=n(),k=s("ul"),k.innerHTML=Ue,ae=n(),_=s("p"),_.textContent=je,ne=n(),c(I.$$.fragment),ie=n(),c(b.$$.fragment),se=n(),O=s("p"),O.textContent=xe,me=n(),c(N.$$.fragment),ce=n(),c(Q.$$.fragment),oe=n(),W=s("p"),W.textContent=ye,pe=n(),c(Y.$$.fragment),he=n(),R=s("p"),R.textContent=Ae,re=n(),c(Z.$$.fragment),ue=n(),c(E.$$.fragment),fe=n(),L=s("p"),this.h()},l(e){const t=We("svelte-u9bgzb",document.head);g=m(t,"META",{name:!0,content:!0}),t.forEach(l),P=i(e),G=m(e,"P",{}),ke(G).forEach(l),V=i(e),o(T.$$.fragment,e),H=i(e),o(d.$$.fragment,e),B=i(e),M=m(e,"P",{"data-svelte-h":!0}),f(M)!=="svelte-i0emib"&&(M.textContent=de),F=i(e),o(w.$$.fragment,e),S=i(e),C=m(e,"P",{"data-svelte-h":!0}),f(C)!=="svelte-q0e2o4"&&(C.textContent=Me),D=i(e),o(J.$$.fragment,e),z=i(e),v=m(e,"P",{"data-svelte-h":!0}),f(v)!=="svelte-v21tje"&&(v.textContent=we),X=i(e),o(U.$$.fragment,e),q=i(e),j=m(e,"P",{"data-svelte-h":!0}),f(j)!=="svelte-ubj59h"&&(j.textContent=Ce),K=i(e),x=m(e,"UL",{"data-svelte-h":!0}),f(x)!=="svelte-rcuru1"&&(x.innerHTML=Je),ee=i(e),o(y.$$.fragment,e),te=i(e),A=m(e,"P",{"data-svelte-h":!0}),f(A)!=="svelte-6piv8e"&&(A.textContent=ve),le=i(e),k=m(e,"UL",{"data-svelte-h":!0}),f(k)!=="svelte-kksjsp"&&(k.innerHTML=Ue),ae=i(e),_=m(e,"P",{"data-svelte-h":!0}),f(_)!=="svelte-1po7rv"&&(_.textContent=je),ne=i(e),o(I.$$.fragment,e),ie=i(e),o(b.$$.fragment,e),se=i(e),O=m(e,"P",{"data-svelte-h":!0}),f(O)!=="svelte-dtxxli"&&(O.textContent=xe),me=i(e),o(N.$$.fragment,e),ce=i(e),o(Q.$$.fragment,e),oe=i(e),W=m(e,"P",{"data-svelte-h":!0}),f(W)!=="svelte-aebkpt"&&(W.textContent=ye),pe=i(e),o(Y.$$.fragment,e),he=i(e),R=m(e,"P",{"data-svelte-h":!0}),f(R)!=="svelte-oib9is"&&(R.textContent=Ae),re=i(e),o(Z.$$.fragment,e),ue=i(e),o(E.$$.fragment,e),fe=i(e),L=m(e,"P",{}),ke(L).forEach(l),this.h()},h(){_e(g,"name","hf:doc:metadata"),_e(g,"content",Ge)},m(e,t){Ye(document.head,g),a(e,P,t),a(e,G,t),a(e,V,t),p(T,e,t),a(e,H,t),p(d,e,t),a(e,B,t),a(e,M,t),a(e,F,t),p(w,e,t),a(e,S,t),a(e,C,t),a(e,D,t),p(J,e,t),a(e,z,t),a(e,v,t),a(e,X,t),p(U,e,t),a(e,q,t),a(e,j,t),a(e,K,t),a(e,x,t),a(e,ee,t),p(y,e,t),a(e,te,t),a(e,A,t),a(e,le,t),a(e,k,t),a(e,ae,t),a(e,_,t),a(e,ne,t),p(I,e,t),a(e,ie,t),p(b,e,t),a(e,se,t),a(e,O,t),a(e,me,t),p(N,e,t),a(e,ce,t),p(Q,e,t),a(e,oe,t),a(e,W,t),a(e,pe,t),p(Y,e,t),a(e,he,t),a(e,R,t),a(e,re,t),p(Z,e,t),a(e,ue,t),p(E,e,t),a(e,fe,t),a(e,L,t),ge=!0},p:be,i(e){ge||(h(T.$$.fragment,e),h(d.$$.fragment,e),h(w.$$.fragment,e),h(J.$$.fragment,e),h(U.$$.fragment,e),h(y.$$.fragment,e),h(I.$$.fragment,e),h(b.$$.fragment,e),h(N.$$.fragment,e),h(Q.$$.fragment,e),h(Y.$$.fragment,e),h(Z.$$.fragment,e),h(E.$$.fragment,e),ge=!0)},o(e){r(T.$$.fragment,e),r(d.$$.fragment,e),r(w.$$.fragment,e),r(J.$$.fragment,e),r(U.$$.fragment,e),r(y.$$.fragment,e),r(I.$$.fragment,e),r(b.$$.fragment,e),r(N.$$.fragment,e),r(Q.$$.fragment,e),r(Y.$$.fragment,e),r(Z.$$.fragment,e),r(E.$$.fragment,e),ge=!1},d(e){e&&(l(P),l(G),l(V),l(H),l(B),l(M),l(F),l(S),l(C),l(D),l(z),l(v),l(X),l(q),l(j),l(K),l(x),l(ee),l(te),l(A),l(le),l(k),l(ae),l(_),l(ne),l(ie),l(se),l(O),l(me),l(ce),l(oe),l(W),l(pe),l(he),l(R),l(re),l(ue),l(fe),l(L)),l(g),u(T,e),u(d,e),u(w,e),u(J,e),u(U,e),u(y,e),u(I,e),u(b,e),u(N,e),u(Q,e),u(Y,e),u(Z,e),u(E,e)}}}const Ge='{"title":"Caching System","local":"caching-system","sections":[{"title":"How It Works","local":"how-it-works","sections":[{"title":"Cache Structure","local":"cache-structure","sections":[],"depth":3},{"title":"Cache Recreation","local":"cache-recreation","sections":[],"depth":3}],"depth":2},{"title":"Using Caching","local":"using-caching","sections":[{"title":"Automatic Caching","local":"automatic-caching","sections":[],"depth":3}],"depth":2},{"title":"Cache Management","local":"cache-management","sections":[{"title":"Clearing Cache","local":"clearing-cache","sections":[],"depth":3}],"depth":2}],"depth":1}';function Le(Te){return Oe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fe extends Ne{constructor(g){super(),Qe(this,g,Le,Ee,Ie,{})}}export{Fe as component};