lighteval/pr_1027/en/_app/immutable/nodes/13.586ff8bf.js

import{s as It,o as Ut,n as Ht}from"../chunks/scheduler.3a17fb72.js";import{S as Lt,i as jt,e as i,s as n,c as b,h as Bt,a as g,d as o,b as l,f as F,g as $,j as h,k as D,l as e,m as v,n as k,t as x,o as T,p as y}from"../chunks/index.093f8863.js";import{C as Ft,H as Dt,E as zt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.45526a62.js";import{D as Q}from"../chunks/Docstring.38455f04.js";import{C as Pt}from"../chunks/CodeBlock.36d3c07b.js";import{E as Vt}from"../chunks/ExampleCodeBlock.ee705a6f.js";function Wt(A){let c,M="Example:",_,d,p;return d=new Pt({props:{code:"dHJhY2tlciUyMCUzRCUyMEV2YWx1YXRpb25UcmFja2VyKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjIuJTJGZXZhbF9yZXN1bHRzJTIyJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwaHViX3Jlc3VsdHNfb3JnJTNEJTIybXktb3JnJTIyJTJDJTBBJTIwJTIwJTIwJTIwc2F2ZV9kZXRhaWxzJTNEVHJ1ZSUwQSklMEElMEElMjMlMjBMb2clMjBldmFsdWF0aW9uJTIwcmVzdWx0cyUwQXRyYWNrZXIubWV0cmljc19sb2dnZXIuYWRkX21ldHJpYyglMjJhY2N1cmFjeSUyMiUyQyUyMDAuODUpJTBBdHJhY2tlci5kZXRhaWxzX2xvZ2dlci5hZGRfZGV0YWlsKHRhc2tfbmFtZSUzRCUyMnFhJTIyJTJDJTIwcHJlZGljdGlvbiUzRCUyMlBhcmlzJTIyKSUwQSUwQSUyMyUyMFNhdmUlMjBhbGwlMjByZXN1bHRzJTBBdHJhY2tlci5zYXZlKCk=",highlighted:`tracker = EvaluationTracker(
    output_dir=<span class="hljs-string">&quot;./eval_results&quot;</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
    hub_results_org=<span class="hljs-string">&quot;my-org&quot;</span>,
    save_details=<span class="hljs-literal">True</span>
)

<span class="hljs-comment"># Log evaluation results</span>
tracker.metrics_logger.add_metric(<span class="hljs-string">&quot;accuracy&quot;</span>, <span class="hljs-number">0.85</span>)
tracker.details_logger.add_detail(task_name=<span class="hljs-string">&quot;qa&quot;</span>, prediction=<span class="hljs-string">&quot;Paris&quot;</span>)

<span class="hljs-comment"># Save all results</span>
tracker.save()`,wrap:!1}}),{c(){c=i("p"),c.textContent=M,_=n(),b(d.$$.fragment)},l(s){c=g(s,"P",{"data-svelte-h":!0}),h(c)!=="svelte-11lpom8"&&(c.textContent=M),_=l(s),$(d.$$.fragment,s)},m(s,m){v(s,c,m),v(s,_,m),k(d,s,m),p=!0},p:Ht,i(s){p||(x(d.$$.fragment,s),p=!0)},o(s){T(d.$$.fragment,s),p=!1},d(s){s&&(o(c),o(_)),y(d,s)}}}function Rt(A){let c,M,_,d,p,s,m,K,a,I,lt,z,$t="Tracks and manages evaluation results, metrics, and logging for model evaluations.",ot,P,kt="The EvaluationTracker coordinates multiple specialized loggers to track different aspects of model evaluation:",st,V,xt="<li>Details Logger (DetailsLogger): Records per-sample evaluation details and predictions</li> <li>Metrics Logger (MetricsLogger): Tracks aggregate evaluation metrics and scores</li> <li>Versions Logger (VersionsLogger): Records task and dataset versions</li> <li>General Config Logger (GeneralConfigLogger): Stores overall evaluation configuration</li> <li>Task Config Logger (TaskConfigLogger): Maintains per-task configuration details</li>",it,W,Tt="The tracker can save results locally and optionally push them to:",gt,R,yt="<li>Hugging Face Hub as datasets</li> <li>TensorBoard for visualization</li> <li>Trackio or Weights &amp; Biases for experiment tracking</li>",ut,C,ct,f,U,dt,Z,Ct="Aggregates and returns all the logger’s experiment information in a dictionary.",pt,q,Et="This function should be used to gather and display said information at the end of an evaluation run.",mt,E,H,ht,S,Jt="Pushes the experiment details (all the model predictions for every step) to the hub.",vt,J,L,_t,X,wt="Fully updates the details repository metadata card for the currently evaluated model",ft,w,j,bt,N,Mt="Saves the experiment information and results to files, and to the hub if requested.",O,B,tt,Y,et;return p=new Ft({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),m=new Dt({props:{title:"EvaluationTracker",local:"lighteval.logging.evaluation_tracker.EvaluationTracker",headingTag:"h1"}}),I=new Q({props:{name:"class lighteval.logging.evaluation_tracker.EvaluationTracker",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker",parameters:[{name:"output_dir",val:": str"},{name:"results_path_template",val:": str | None = None"},{name:"save_details",val:": bool = True"},{name:"push_to_hub",val:": bool = False"},{name:"push_to_tensorboard",val:": bool = False"},{name:"hub_results_org",val:": str | None = ''"},{name:"tensorboard_metric_prefix",val:": str = 'eval'"},{name:"public",val:": bool = False"},{name:"nanotron_run_info",val:": GeneralArgs = None"},{name:"use_wandb",val:": bool = False"}],parametersDescription:[{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.output_dir",description:"<strong>output_dir</strong> (str) &#x2014; Local directory to save evaluation results and logs",name:"output_dir"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.results_path_template",description:`<strong>results_path_template</strong> (str, optional) &#x2014; Template for results directory structure.
Example: &#x201C;{output<em>dir}/results/{org}</em>{model}&#x201D;`,name:"results_path_template"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.save_details",description:"<strong>save_details</strong> (bool, defaults to True) &#x2014; Whether to save detailed evaluation records",name:"save_details"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.push_to_hub",description:"<strong>push_to_hub</strong> (bool, defaults to False) &#x2014; Whether to push results to HF Hub",name:"push_to_hub"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.push_to_tensorboard",description:"<strong>push_to_tensorboard</strong> (bool, defaults to False) &#x2014; Whether to push metrics to TensorBoard",name:"push_to_tensorboard"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.hub_results_org",description:"<strong>hub_results_org</strong> (str, optional) &#x2014; HF Hub organization to push results to",name:"hub_results_org"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.tensorboard_metric_prefix",description:"<strong>tensorboard_metric_prefix</strong> (str, defaults to &#x201C;eval&#x201D;) &#x2014; Prefix for TensorBoard metrics",name:"tensorboard_metric_prefix"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.public",description:"<strong>public</strong> (bool, defaults to False) &#x2014; Whether to make Hub datasets public",name:"public"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.nanotron_run_info",description:"<strong>nanotron_run_info</strong> (GeneralArgs, optional) &#x2014; Nanotron model run information",name:"nanotron_run_info"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.use_wandb",description:"<strong>use_wandb</strong> (bool, defaults to False) &#x2014; Whether to log to Weights &amp; Biases or Trackio if available",name:"use_wandb"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/logging/evaluation_tracker.py#L95"}}),C=new Vt({props:{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.example",$$slots:{default:[Wt]},$$scope:{ctx:A}}}),U=new Q({props:{name:"generate_final_dict",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.generate_final_dict",parameters:[],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/logging/evaluation_tracker.py#L363",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Dictionary containing all experiment information including config, results, versions, and summaries</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>dict</p>
`}}),H=new Q({props:{name:"push_to_hub",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.push_to_hub",parameters:[{name:"date_id",val:": str"},{name:"details",val:": dict"},{name:"results_dict",val:": dict"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/logging/evaluation_tracker.py#L387"}}),L=new Q({props:{name:"recreate_metadata_card",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.recreate_metadata_card",parameters:[{name:"repo_id",val:": str"}],parametersDescription:[{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.recreate_metadata_card.repo_id",description:"<strong>repo_id</strong> (str) &#x2014; Details dataset repository path on the hub (<code>org/dataset</code>)",name:"repo_id"}],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/logging/evaluation_tracker.py#L454"}}),j=new Q({props:{name:"save",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.save",parameters:[],source:"https://github.com/huggingface/lighteval/blob/vr_1027/src/lighteval/logging/evaluation_tracker.py#L247"}}),B=new zt({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/package_reference/evaluation_tracker.mdx"}}),{c(){c=i("meta"),M=n(),_=i("p"),d=n(),b(p.$$.fragment),s=n(),b(m.$$.fragment),K=n(),a=i("div"),b(I.$$.fragment),lt=n(),z=i("p"),z.textContent=$t,ot=n(),P=i("p"),P.textContent=kt,st=n(),V=i("ul"),V.innerHTML=xt,it=n(),W=i("p"),W.textContent=Tt,gt=n(),R=i("ul"),R.innerHTML=yt,ut=n(),b(C.$$.fragment),ct=n(),f=i("div"),b(U.$$.fragment),dt=n(),Z=i("p"),Z.textContent=Ct,pt=n(),q=i("p"),q.textContent=Et,mt=n(),E=i("div"),b(H.$$.fragment),ht=n(),S=i("p"),S.textContent=Jt,vt=n(),J=i("div"),b(L.$$.fragment),_t=n(),X=i("p"),X.textContent=wt,ft=n(),w=i("div"),b(j.$$.fragment),bt=n(),N=i("p"),N.textContent=Mt,O=n(),b(B.$$.fragment),tt=n(),Y=i("p"),this.h()},l(t){const u=Bt("svelte-u9bgzb",document.head);c=g(u,"META",{name:!0,content:!0}),u.forEach(o),M=l(t),_=g(t,"P",{}),F(_).forEach(o),d=l(t),$(p.$$.fragment,t),s=l(t),$(m.$$.fragment,t),K=l(t),a=g(t,"DIV",{class:!0});var r=F(a);$(I.$$.fragment,r),lt=l(r),z=g(r,"P",{"data-svelte-h":!0}),h(z)!=="svelte-1a7u8o3"&&(z.textContent=$t),ot=l(r),P=g(r,"P",{"data-svelte-h":!0}),h(P)!=="svelte-had3jk"&&(P.textContent=kt),st=l(r),V=g(r,"UL",{"data-svelte-h":!0}),h(V)!=="svelte-1gbvzvg"&&(V.innerHTML=xt),it=l(r),W=g(r,"P",{"data-svelte-h":!0}),h(W)!=="svelte-cw347r"&&(W.textContent=Tt),gt=l(r),R=g(r,"UL",{"data-svelte-h":!0}),h(R)!=="svelte-1k0oe7e"&&(R.innerHTML=yt),ut=l(r),$(C.$$.fragment,r),ct=l(r),f=g(r,"DIV",{class:!0});var G=F(f);$(U.$$.fragment,G),dt=l(G),Z=g(G,"P",{"data-svelte-h":!0}),h(Z)!=="svelte-qqp5v"&&(Z.textContent=Ct),pt=l(G),q=g(G,"P",{"data-svelte-h":!0}),h(q)!=="svelte-1ke5vz1"&&(q.textContent=Et),G.forEach(o),mt=l(r),E=g(r,"DIV",{class:!0});var at=F(E);$(H.$$.fragment,at),ht=l(at),S=g(at,"P",{"data-svelte-h":!0}),h(S)!=="svelte-1lj091"&&(S.textContent=Jt),at.forEach(o),vt=l(r),J=g(r,"DIV",{class:!0});var rt=F(J);$(L.$$.fragment,rt),_t=l(rt),X=g(rt,"P",{"data-svelte-h":!0}),h(X)!=="svelte-1l1nn02"&&(X.textContent=wt),rt.forEach(o),ft=l(r),w=g(r,"DIV",{class:!0});var nt=F(w);$(j.$$.fragment,nt),bt=l(nt),N=g(nt,"P",{"data-svelte-h":!0}),h(N)!=="svelte-1jp5uy4"&&(N.textContent=Mt),nt.forEach(o),r.forEach(o),O=l(t),$(B.$$.fragment,t),tt=l(t),Y=g(t,"P",{}),F(Y).forEach(o),this.h()},h(){D(c,"name","hf:doc:metadata"),D(c,"content",Zt),D(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),D(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),D(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),D(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),D(a,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,u){e(document.head,c),v(t,M,u),v(t,_,u),v(t,d,u),k(p,t,u),v(t,s,u),k(m,t,u),v(t,K,u),v(t,a,u),k(I,a,null),e(a,lt),e(a,z),e(a,ot),e(a,P),e(a,st),e(a,V),e(a,it),e(a,W),e(a,gt),e(a,R),e(a,ut),k(C,a,null),e(a,ct),e(a,f),k(U,f,null),e(f,dt),e(f,Z),e(f,pt),e(f,q),e(a,mt),e(a,E),k(H,E,null),e(E,ht),e(E,S),e(a,vt),e(a,J),k(L,J,null),e(J,_t),e(J,X),e(a,ft),e(a,w),k(j,w,null),e(w,bt),e(w,N),v(t,O,u),k(B,t,u),v(t,tt,u),v(t,Y,u),et=!0},p(t,[u]){const r={};u&2&&(r.$$scope={dirty:u,ctx:t}),C.$set(r)},i(t){et||(x(p.$$.fragment,t),x(m.$$.fragment,t),x(I.$$.fragment,t),x(C.$$.fragment,t),x(U.$$.fragment,t),x(H.$$.fragment,t),x(L.$$.fragment,t),x(j.$$.fragment,t),x(B.$$.fragment,t),et=!0)},o(t){T(p.$$.fragment,t),T(m.$$.fragment,t),T(I.$$.fragment,t),T(C.$$.fragment,t),T(U.$$.fragment,t),T(H.$$.fragment,t),T(L.$$.fragment,t),T(j.$$.fragment,t),T(B.$$.fragment,t),et=!1},d(t){t&&(o(M),o(_),o(d),o(s),o(K),o(a),o(O),o(tt),o(Y)),o(c),y(p,t),y(m,t),y(I),y(C),y(U),y(H),y(L),y(j),y(B,t)}}}const Zt='{"title":"EvaluationTracker","local":"lighteval.logging.evaluation_tracker.EvaluationTracker","sections":[],"depth":1}';function qt(A){return Ut(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class At extends Lt{constructor(c){super(),jt(this,c,qt,Rt,It,{})}}export{At as component};