lighteval/pr_1326/en/_app/immutable/nodes/14.BDPLPkYW.js

import"../chunks/DsnmJJEf.js";import{i as I,h as j,C as B,H as F,D as e,E as H,s as L,a as C}from"../chunks/CyvF58-O.js";import{p as W,o as D,s as a,f as T,a as u,b as R,d as t,c as d,n as r,r as o}from"../chunks/DfHjNWj2.js";import{E as Z}from"../chunks/BW1mQolG.js";const z='{"title":"EvaluationTracker","local":"lighteval.logging.evaluation_tracker.EvaluationTracker","sections":[],"depth":1}';var X=d('<meta name="hf:doc:metadata"/>'),N=d("<p>Example:</p> <!>",1),S=d('<p></p> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Tracks and manages evaluation results, metrics, and logging for model evaluations.</p> <p>The EvaluationTracker coordinates multiple specialized loggers to track different aspects of model evaluation:</p> <ul><li>Details Logger (DetailsLogger): Records per-sample evaluation details and predictions</li> <li>Metrics Logger (MetricsLogger): Tracks aggregate evaluation metrics and scores</li> <li>Versions Logger (VersionsLogger): Records task and dataset versions</li> <li>General Config Logger (GeneralConfigLogger): Stores overall evaluation configuration</li> <li>Task Config Logger (TaskConfigLogger): Maintains per-task configuration details</li></ul> <p>The tracker can save results locally and optionally push them to:</p> <ul><li>Hugging Face Hub as datasets</li> <li>TensorBoard for visualization</li> <li>Trackio or Weights & Biases for experiment tracking</li></ul> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Aggregates and returns all the logger’s experiment information in a dictionary.</p> <p>This function should be used to gather and display said information at the end of an evaluation run.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Pushes the experiment details (all the model predictions for every step) to the hub.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Fully updates the details repository metadata card for the currently evaluated model</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Saves the experiment information and results to files, and to the hub if requested.</p></div></div> <!> <p></p>',1);function P(f,y){W(y,!1),D(()=>{new URLSearchParams(window.location.search).get("fw")}),I();var p=S();j("7svjjb",l=>{var c=X();L(c,"content",z),u(l,c)});var v=a(T(p),2);B(v,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var h=a(v,2);F(h,{title:"EvaluationTracker",local:"lighteval.logging.evaluation_tracker.EvaluationTracker",headingTag:"h1"});var n=a(h,2),m=t(n);e(m,{name:"class lighteval.logging.evaluation_tracker.EvaluationTracker",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker",source:"https://github.com/huggingface/lighteval/blob/vr_1326/src/lighteval/logging/evaluation_tracker.py#L95",parameters:[{name:"output_dir",val:": str"},{name:"results_path_template",val:": str | None = None"},{name:"save_details",val:": bool = True"},{name:"push_to_hub",val:": bool = False"},{name:"push_to_tensorboard",val:": bool = False"},{name:"hub_results_org",val:": str | None = ''"},{name:"tensorboard_metric_prefix",val:": str = 'eval'"},{name:"public",val:": bool = False"},{name:"nanotron_run_info",val:": GeneralArgs = None"},{name:"use_wandb",val:": bool = False"}],parametersDescription:[{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.output_dir",description:"<strong>output_dir</strong> (str) &#x2014; Local directory to save evaluation results and logs",name:"output_dir"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.results_path_template",description:`<strong>results_path_template</strong> (str, optional) &#x2014; Template for results directory structure.
Example: &#x201C;{output<em>dir}/results/{org}</em>{model}&#x201D;`,name:"results_path_template"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.save_details",description:"<strong>save_details</strong> (bool, defaults to True) &#x2014; Whether to save detailed evaluation records",name:"save_details"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.push_to_hub",description:"<strong>push_to_hub</strong> (bool, defaults to False) &#x2014; Whether to push results to HF Hub",name:"push_to_hub"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.push_to_tensorboard",description:"<strong>push_to_tensorboard</strong> (bool, defaults to False) &#x2014; Whether to push metrics to TensorBoard",name:"push_to_tensorboard"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.hub_results_org",description:"<strong>hub_results_org</strong> (str, optional) &#x2014; HF Hub organization to push results to",name:"hub_results_org"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.tensorboard_metric_prefix",description:"<strong>tensorboard_metric_prefix</strong> (str, defaults to &#x201C;eval&#x201D;) &#x2014; Prefix for TensorBoard metrics",name:"tensorboard_metric_prefix"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.public",description:"<strong>public</strong> (bool, defaults to False) &#x2014; Whether to make Hub datasets public",name:"public"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.nanotron_run_info",description:"<strong>nanotron_run_info</strong> (GeneralArgs, optional) &#x2014; Nanotron model run information",name:"nanotron_run_info"},{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.use_wandb",description:"<strong>use_wandb</strong> (bool, defaults to False) &#x2014; Whether to log to Weights &amp; Biases or Trackio if available",name:"use_wandb"}]});var _=a(m,12);Z(_,{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.example",children:(l,c)=>{var k=N(),U=a(T(k),2);C(U,{code:"dHJhY2tlciUyMCUzRCUyMEV2YWx1YXRpb25UcmFja2VyKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjIuJTJGZXZhbF9yZXN1bHRzJTIyJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwaHViX3Jlc3VsdHNfb3JnJTNEJTIybXktb3JnJTIyJTJDJTBBJTIwJTIwJTIwJTIwc2F2ZV9kZXRhaWxzJTNEVHJ1ZSUwQSklMEElMEElMjMlMjBMb2clMjBldmFsdWF0aW9uJTIwcmVzdWx0cyUwQXRyYWNrZXIubWV0cmljc19sb2dnZXIuYWRkX21ldHJpYyglMjJhY2N1cmFjeSUyMiUyQyUyMDAuODUpJTBBdHJhY2tlci5kZXRhaWxzX2xvZ2dlci5hZGRfZGV0YWlsKHRhc2tfbmFtZSUzRCUyMnFhJTIyJTJDJTIwcHJlZGljdGlvbiUzRCUyMlBhcmlzJTIyKSUwQSUwQSUyMyUyMFNhdmUlMjBhbGwlMjByZXN1bHRzJTBBdHJhY2tlci5zYXZlKCk=",highlighted:`tracker = EvaluationTracker(
    output_dir=<span class="hljs-string">&quot;./eval_results&quot;</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
    hub_results_org=<span class="hljs-string">&quot;my-org&quot;</span>,
    save_details=<span class="hljs-literal">True</span>
)

<span class="hljs-comment"># Log evaluation results</span>
tracker.metrics_logger.add_metric(<span class="hljs-string">&quot;accuracy&quot;</span>, <span class="hljs-number">0.85</span>)
tracker.details_logger.add_detail(task_name=<span class="hljs-string">&quot;qa&quot;</span>, prediction=<span class="hljs-string">&quot;Paris&quot;</span>)

<span class="hljs-comment"># Save all results</span>
tracker.save()`,lang:"python",wrap:!1}),u(l,k)},$$slots:{default:!0}});var i=a(_,2),x=t(i);e(x,{name:"generate_final_dict",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.generate_final_dict",source:"https://github.com/huggingface/lighteval/blob/vr_1326/src/lighteval/logging/evaluation_tracker.py#L365",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Dictionary containing all experiment information including config, results, versions, and summaries</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>dict</p>
`}),r(4),o(i);var s=a(i,2),J=t(s);e(J,{name:"push_to_hub",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.push_to_hub",source:"https://github.com/huggingface/lighteval/blob/vr_1326/src/lighteval/logging/evaluation_tracker.py#L389",parameters:[{name:"date_id",val:": str"},{name:"details",val:": dict"},{name:"results_dict",val:": dict"}]}),r(2),o(s);var g=a(s,2),E=t(g);e(E,{name:"recreate_metadata_card",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.recreate_metadata_card",source:"https://github.com/huggingface/lighteval/blob/vr_1326/src/lighteval/logging/evaluation_tracker.py#L456",parameters:[{name:"repo_id",val:": str"}],parametersDescription:[{anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.recreate_metadata_card.repo_id",description:"<strong>repo_id</strong> (str) &#x2014; Details dataset repository path on the hub (<code>org/dataset</code>)",name:"repo_id"}]}),r(2),o(g);var b=a(g,2),M=t(b);e(M,{name:"save",anchor:"lighteval.logging.evaluation_tracker.EvaluationTracker.save",source:"https://github.com/huggingface/lighteval/blob/vr_1326/src/lighteval/logging/evaluation_tracker.py#L249",parameters:[]}),r(2),o(b),o(n);var w=a(n,2);H(w,{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/package_reference/evaluation_tracker.mdx"}),r(2),u(f,p),R()}export{P as component};