Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Running ML Experiments with Agents","local":"running-ml-experiments-with-agents","sections":[{"title":"Why Trackio for Agents?","local":"why-trackio-for-agents","sections":[],"depth":2},{"title":"The Agent Feedback Loop","local":"the-agent-feedback-loop","sections":[],"depth":2},{"title":"Step 1: Structuring Training Code for Agents","local":"step-1-structuring-training-code-for-agents","sections":[{"title":"Guidelines for Agent-Friendly Alerts","local":"guidelines-for-agent-friendly-alerts","sections":[],"depth":3}],"depth":2},{"title":"Step 2: Launching the Script","local":"step-2-launching-the-script","sections":[],"depth":2},{"title":"Step 3: Monitoring During Training","local":"step-3-monitoring-during-training","sections":[{"title":"Polling via CLI","local":"polling-via-cli","sections":[],"depth":3},{"title":"Polling via Python API","local":"polling-via-python-api","sections":[],"depth":3},{"title":"Polling via MCP","local":"polling-via-mcp","sections":[],"depth":3}],"depth":2},{"title":"Step 4: Reading Results and Deciding Next Steps","local":"step-4-reading-results-and-deciding-next-steps","sections":[],"depth":2},{"title":"Step 5: Iterating on Hyperparameters","local":"step-5-iterating-on-hyperparameters","sections":[],"depth":2},{"title":"Comparing Runs","local":"comparing-runs","sections":[],"depth":2},{"title":"Sending Alerts to the Agent via Webhooks","local":"sending-alerts-to-the-agent-via-webhooks","sections":[],"depth":2},{"title":"Full Example: Autonomous Hyperparameter Search","local":"full-example-autonomous-hyperparameter-search","sections":[],"depth":2},{"title":"Using Alerts with Transformers and TRL","local":"using-alerts-with-transformers-and-trl","sections":[{"title":"Transformers","local":"transformers","sections":[],"depth":3},{"title":"TRL","local":"trl","sections":[],"depth":3},{"title":"Tips for Agents Writing Alert Callbacks","local":"tips-for-agents-writing-alert-callbacks","sections":[],"depth":3}],"depth":2},{"title":"Best Practices Summary","local":"best-practices-summary","sections":[],"depth":2}],"depth":1}"/> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/entry/start.B75uqjYQ.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/chunks/4xzTs_ti.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/chunks/c5epZbRJ.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/entry/app.D_-Gc0EO.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/chunks/CTPvGMXu.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/chunks/ChRZvuIG.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/chunks/CldSabhm.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/nodes/0.BDlkYnFr.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/chunks/DoyOpfDJ.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/nodes/15.B17LesCk.js" rel="modulepreload"> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/chunks/BQnA81uy.js" rel="modulepreload"> | |
| <!--1u3tfgr--><meta name="hf:doc:metadata" content="{"title":"Running ML Experiments with Agents","local":"running-ml-experiments-with-agents","sections":[{"title":"Why Trackio for Agents?","local":"why-trackio-for-agents","sections":[],"depth":2},{"title":"The Agent Feedback Loop","local":"the-agent-feedback-loop","sections":[],"depth":2},{"title":"Step 1: Structuring Training Code for Agents","local":"step-1-structuring-training-code-for-agents","sections":[{"title":"Guidelines for Agent-Friendly Alerts","local":"guidelines-for-agent-friendly-alerts","sections":[],"depth":3}],"depth":2},{"title":"Step 2: Launching the Script","local":"step-2-launching-the-script","sections":[],"depth":2},{"title":"Step 3: Monitoring During Training","local":"step-3-monitoring-during-training","sections":[{"title":"Polling via CLI","local":"polling-via-cli","sections":[],"depth":3},{"title":"Polling via Python API","local":"polling-via-python-api","sections":[],"depth":3},{"title":"Polling via MCP","local":"polling-via-mcp","sections":[],"depth":3}],"depth":2},{"title":"Step 4: Reading Results and Deciding Next Steps","local":"step-4-reading-results-and-deciding-next-steps","sections":[],"depth":2},{"title":"Step 5: Iterating on Hyperparameters","local":"step-5-iterating-on-hyperparameters","sections":[],"depth":2},{"title":"Comparing Runs","local":"comparing-runs","sections":[],"depth":2},{"title":"Sending Alerts to the Agent via Webhooks","local":"sending-alerts-to-the-agent-via-webhooks","sections":[],"depth":2},{"title":"Full Example: Autonomous Hyperparameter Search","local":"full-example-autonomous-hyperparameter-search","sections":[],"depth":2},{"title":"Using Alerts with Transformers and TRL","local":"using-alerts-with-transformers-and-trl","sections":[{"title":"Transformers","local":"transformers","sections":[],"depth":3},{"title":"TRL","local":"trl","sections":[],"depth":3},{"title":"Tips for Agents Writing Alert Callbacks","local":"tips-for-agents-writing-alert-callbacks","sections":[],"depth":3}],"depth":2},{"title":"Best Practices Summary","local":"best-practices-summary","sections":[],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/trackio/pr_672/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="running-ml-experiments-with-agents" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#running-ml-experiments-with-agents"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Running ML Experiments with Agents</span></h1><!--]--><!----> <p>Trackio is designed from the ground up to support autonomous ML experimentation driven by LLM agents. This guide covers how to structure your training code, configure alerts, and wire up the feedback loop so that an agent can launch experiments, monitor them, react to problems, and iterate on hyperparameters — all without human intervention.</p> <!--[1--><h2 class="relative group"><a id="why-trackio-for-agents" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#why-trackio-for-agents"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Why Trackio for Agents?</span></h2><!--]--><!----> <p>Most experiment trackers assume a human is watching a dashboard. Trackio gives agents first-class programmatic access through multiple channels:</p> <ul><li><strong>CLI with JSON output</strong> — every <code>trackio</code> command supports <code>--json</code>, making it trivial to parse in any language</li> <li><strong>Python API</strong> — <code>trackio.Api()</code> provides direct access to runs, metrics, and alerts</li> <li><strong>HTTP API / MCP</strong> — the Trackio dashboard exposes Gradio API endpoints, and can run as an MCP server so LLMs can call tools directly</li> <li><strong>Alerts with webhooks</strong> — agents can insert <code>trackio.alert()</code> calls like print statements and poll for them later</li> <li><strong><code>since</code> filtering</strong> — efficient polling for new alerts without re-reading the full history</li></ul> <!--[1--><h2 class="relative group"><a id="the-agent-feedback-loop" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#the-agent-feedback-loop"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>The Agent Feedback Loop</span></h2><!--]--><!----> <p>A typical autonomous experiment loop looks like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class=" "><!---->┌─────────────────────────────────────┐ | |
| │ <span class="hljs-number">1.</span> Agent writes training <span class="hljs-keyword">script</span> │ | |
| │ <span class="hljs-keyword">with</span> trackio.init(), <span class="hljs-built_in">log</span>(), │ | |
| │ <span class="hljs-keyword">and</span> alert() calls │ | |
| ├─────────────────────────────────────┤ | |
| │ <span class="hljs-number">2.</span> Agent launches <span class="hljs-keyword">the</span> <span class="hljs-keyword">script</span> │ | |
| ├─────────────────────────────────────┤ | |
| │ <span class="hljs-number">3.</span> Agent polls <span class="hljs-keyword">for</span> alerts <span class="hljs-keyword">and</span> │ | |
| │ metrics <span class="hljs-keyword">while</span> training runs │ | |
| ├─────────────────────────────────────┤ | |
| │ <span class="hljs-number">4.</span> Agent reads results, decides │ | |
| │ next hyperparameters │ | |
| ├─────────────────────────────────────┤ | |
| │ <span class="hljs-number">5.</span> Agent writes a new <span class="hljs-keyword">script</span> │ | |
| │ <span class="hljs-keyword">and</span> repeats <span class="hljs-keyword">from</span> step <span class="hljs-number">1</span> │ | |
| └─────────────────────────────────────┘<!----></pre></div><!----> <p>The rest of this guide walks through each step with concrete code.</p> <!--[1--><h2 class="relative group"><a id="step-1-structuring-training-code-for-agents" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#step-1-structuring-training-code-for-agents"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Step 1: Structuring Training Code for Agents</span></h2><!--]--><!----> <p>The key insight is that an agent can insert <code>trackio.alert()</code> calls the same way it would insert <code>print()</code> statements for debugging. Alerts are more structured than print statements: they have a title, optional description, severity level, and are automatically persisted and queryable.</p> <p>Here is a training script written the way an agent would write it, with alerts at every decision point:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> trackio | |
| <span class="hljs-keyword">import</span> torch | |
| trackio.init( | |
| project=<span class="hljs-string">"cifar10-hparam-search"</span>, | |
| name=<span class="hljs-string">"run-lr0.001-bs64"</span>, | |
| config={ | |
| <span class="hljs-string">"learning_rate"</span>: <span class="hljs-number">0.001</span>, | |
| <span class="hljs-string">"batch_size"</span>: <span class="hljs-number">64</span>, | |
| <span class="hljs-string">"optimizer"</span>: <span class="hljs-string">"adamw"</span>, | |
| <span class="hljs-string">"epochs"</span>: <span class="hljs-number">50</span>, | |
| <span class="hljs-string">"weight_decay"</span>: <span class="hljs-number">1e-4</span>, | |
| }, | |
| ) | |
| model = build_model() | |
| optimizer = torch.optim.AdamW( | |
| model.parameters(), lr=<span class="hljs-number">0.001</span>, weight_decay=<span class="hljs-number">1e-4</span> | |
| ) | |
| best_val_loss = <span class="hljs-built_in">float</span>(<span class="hljs-string">"inf"</span>) | |
| patience_counter = <span class="hljs-number">0</span> | |
| patience_limit = <span class="hljs-number">5</span> | |
| <span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">50</span>): | |
| train_loss = train_one_epoch(model, train_loader, optimizer) | |
| val_loss, val_acc = evaluate(model, val_loader) | |
| trackio.log({ | |
| <span class="hljs-string">"train_loss"</span>: train_loss, | |
| <span class="hljs-string">"val_loss"</span>: val_loss, | |
| <span class="hljs-string">"val_accuracy"</span>: val_acc, | |
| <span class="hljs-string">"learning_rate"</span>: optimizer.param_groups[<span class="hljs-number">0</span>][<span class="hljs-string">"lr"</span>], | |
| }) | |
| <span class="hljs-comment"># Alert on loss divergence</span> | |
| <span class="hljs-keyword">if</span> train_loss > <span class="hljs-number">10.0</span>: | |
| trackio.alert( | |
| title=<span class="hljs-string">"Training diverged"</span>, | |
| text=<span class="hljs-string">f"train_loss=<span class="hljs-subst">{train_loss:<span class="hljs-number">.4</span>f}</span> at epoch <span class="hljs-subst">{epoch}</span>. "</span> | |
| <span class="hljs-string">"Learning rate is likely too high."</span>, | |
| level=trackio.AlertLevel.ERROR, | |
| ) | |
| <span class="hljs-keyword">break</span> | |
| <span class="hljs-comment"># Alert when validation stops improving</span> | |
| <span class="hljs-keyword">if</span> val_loss < best_val_loss: | |
| best_val_loss = val_loss | |
| patience_counter = <span class="hljs-number">0</span> | |
| torch.save(model.state_dict(), <span class="hljs-string">"best_model.pt"</span>) | |
| <span class="hljs-keyword">else</span>: | |
| patience_counter += <span class="hljs-number">1</span> | |
| <span class="hljs-keyword">if</span> patience_counter >= patience_limit: | |
| trackio.alert( | |
| title=<span class="hljs-string">"Early stopping triggered"</span>, | |
| text=<span class="hljs-string">f"No improvement for <span class="hljs-subst">{patience_limit}</span> epochs. "</span> | |
| <span class="hljs-string">f"Best val_loss=<span class="hljs-subst">{best_val_loss:<span class="hljs-number">.4</span>f}</span> at epoch "</span> | |
| <span class="hljs-string">f"<span class="hljs-subst">{epoch - patience_limit}</span>."</span>, | |
| level=trackio.AlertLevel.WARN, | |
| ) | |
| <span class="hljs-keyword">break</span> | |
| <span class="hljs-comment"># Alert on overfitting</span> | |
| <span class="hljs-keyword">if</span> epoch > <span class="hljs-number">10</span> <span class="hljs-keyword">and</span> (train_loss < <span class="hljs-number">0.1</span> <span class="hljs-keyword">and</span> val_loss > <span class="hljs-number">1.5</span>): | |
| trackio.alert( | |
| title=<span class="hljs-string">"Overfitting detected"</span>, | |
| text=<span class="hljs-string">f"train_loss=<span class="hljs-subst">{train_loss:<span class="hljs-number">.4</span>f}</span> vs val_loss=<span class="hljs-subst">{val_loss:<span class="hljs-number">.4</span>f}</span>. "</span> | |
| <span class="hljs-string">"Consider adding regularization or reducing model capacity."</span>, | |
| level=trackio.AlertLevel.WARN, | |
| ) | |
| <span class="hljs-comment"># Alert on good results</span> | |
| <span class="hljs-keyword">if</span> val_acc > <span class="hljs-number">0.93</span>: | |
| trackio.alert( | |
| title=<span class="hljs-string">"High accuracy reached"</span>, | |
| text=<span class="hljs-string">f"val_accuracy=<span class="hljs-subst">{val_acc:<span class="hljs-number">.4</span>f}</span> at epoch <span class="hljs-subst">{epoch}</span>."</span>, | |
| level=trackio.AlertLevel.INFO, | |
| ) | |
| trackio.alert( | |
| title=<span class="hljs-string">"Training complete"</span>, | |
| text=<span class="hljs-string">f"Final val_loss=<span class="hljs-subst">{val_loss:<span class="hljs-number">.4</span>f}</span>, val_accuracy=<span class="hljs-subst">{val_acc:<span class="hljs-number">.4</span>f}</span>. "</span> | |
| <span class="hljs-string">f"Best val_loss=<span class="hljs-subst">{best_val_loss:<span class="hljs-number">.4</span>f}</span>."</span>, | |
| level=trackio.AlertLevel.INFO, | |
| ) | |
| trackio.finish()<!----></pre></div><!----> <p>The alerts here are not just for humans — they are structured signals that the agent will read later to decide what to do next.</p> <!--[2--><h3 class="relative group"><a id="guidelines-for-agent-friendly-alerts" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#guidelines-for-agent-friendly-alerts"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Guidelines for Agent-Friendly Alerts</span></h3><!--]--><!----> <ul><li><strong>Use <code>ERROR</code> for conditions that should stop the run</strong> (divergence, NaN loss, out of memory).</li> <li><strong>Use <code>WARN</code> for conditions that suggest the hyperparameters should change</strong> (overfitting, early stopping, slow convergence).</li> <li><strong>Use <code>INFO</code> for milestones</strong> (training complete, high accuracy reached, checkpoint saved).</li> <li><strong>Include numeric values in <code>text</code></strong> so the agent can parse them and reason about them.</li> <li><strong>Include actionable suggestions in <code>text</code></strong> (e.g., “Learning rate is likely too high”) to help the agent decide what to change.</li></ul> <!--[1--><h2 class="relative group"><a id="step-2-launching-the-script" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#step-2-launching-the-script"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Step 2: Launching the Script</span></h2><!--]--><!----> <p>The agent can launch the script in a subprocess or background process. In most agent frameworks, this is as simple as running a shell command:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->python train.py<!----></pre></div><!----> <p>If the agent is using an MCP-connected LLM, it can also launch scripts through a shell tool.</p> <!--[1--><h2 class="relative group"><a id="step-3-monitoring-during-training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#step-3-monitoring-during-training"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Step 3: Monitoring During Training</span></h2><!--]--><!----> <p>While training runs, the agent polls for alerts and metrics. The <code>--since</code> flag makes this efficient — only new alerts since the last check are returned.</p> <!--[2--><h3 class="relative group"><a id="polling-via-cli" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#polling-via-cli"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Polling via CLI</span></h3><!--]--><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># First poll — get all alerts</span> | |
| trackio get alerts --project <span class="hljs-string">"cifar10-hparam-search"</span> --run <span class="hljs-string">"run-lr0.001-bs64"</span> --json | |
| <span class="hljs-comment"># Subsequent polls — only new alerts</span> | |
| trackio get alerts --project <span class="hljs-string">"cifar10-hparam-search"</span> --since <span class="hljs-string">"2025-06-15T14:30:00"</span> --json<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="polling-via-python-api" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#polling-via-python-api"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Polling via Python API</span></h3><!--]--><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> trackio | |
| <span class="hljs-keyword">from</span> datetime <span class="hljs-keyword">import</span> datetime, timezone | |
| api = trackio.Api() | |
| last_check = <span class="hljs-literal">None</span> | |
| <span class="hljs-keyword">while</span> training_is_running(): | |
| alerts = api.alerts( | |
| <span class="hljs-string">"cifar10-hparam-search"</span>, | |
| run=<span class="hljs-string">"run-lr0.001-bs64"</span>, | |
| since=last_check, | |
| ) | |
| <span class="hljs-keyword">if</span> alerts: | |
| last_check = alerts[<span class="hljs-number">0</span>][<span class="hljs-string">"timestamp"</span>] <span class="hljs-comment"># alerts are sorted newest-first</span> | |
| <span class="hljs-keyword">for</span> a <span class="hljs-keyword">in</span> alerts: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"[<span class="hljs-subst">{a[<span class="hljs-string">'level'</span>].upper()}</span>] <span class="hljs-subst">{a[<span class="hljs-string">'title'</span>]}</span>: <span class="hljs-subst">{a[<span class="hljs-string">'text'</span>]}</span>"</span>) | |
| <span class="hljs-keyword">if</span> a[<span class="hljs-string">"level"</span>] == <span class="hljs-string">"error"</span>: | |
| <span class="hljs-comment"># Stop the run and re-plan</span> | |
| handle_error(a) | |
| time.sleep(<span class="hljs-number">30</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="polling-via-mcp" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#polling-via-mcp"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Polling via MCP</span></h3><!--]--><!----> <p>If the Trackio dashboard is running as an MCP server, the agent can call the <code>get_alerts</code> tool directly through the MCP protocol. See <a href="api_mcp_server">Trackio as an API and MCP Server</a> for setup instructions.</p> <!--[1--><h2 class="relative group"><a id="step-4-reading-results-and-deciding-next-steps" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#step-4-reading-results-and-deciding-next-steps"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Step 4: Reading Results and Deciding Next Steps</span></h2><!--]--><!----> <p>After a run finishes, the agent reads the full results:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># Get run summary (metrics, config, step count)</span> | |
| trackio get run --project <span class="hljs-string">"cifar10-hparam-search"</span> --run <span class="hljs-string">"run-lr0.001-bs64"</span> --json | |
| <span class="hljs-comment"># Get the loss curve</span> | |
| trackio get metric --project <span class="hljs-string">"cifar10-hparam-search"</span> --run <span class="hljs-string">"run-lr0.001-bs64"</span> --metric <span class="hljs-string">"val_loss"</span> --json | |
| <span class="hljs-comment"># Get all alerts (the most useful signal for decision-making)</span> | |
| trackio get alerts --project <span class="hljs-string">"cifar10-hparam-search"</span> --run <span class="hljs-string">"run-lr0.001-bs64"</span> --json<!----></pre></div><!----> <p>The alerts provide a concise summary of what happened during training. Instead of parsing thousands of metric values, the agent can read a handful of alerts to understand:</p> <ul><li>Did the run diverge? → Lower the learning rate</li> <li>Did it overfit? → Add regularization or reduce capacity</li> <li>Did it early-stop? → The hyperparameters are in the right ballpark but might need tuning</li> <li>Did it reach high accuracy? → This configuration is working well</li></ul> <!--[1--><h2 class="relative group"><a id="step-5-iterating-on-hyperparameters" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#step-5-iterating-on-hyperparameters"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Step 5: Iterating on Hyperparameters</span></h2><!--]--><!----> <p>The agent uses the alerts from the previous run to choose new hyperparameters. Here is a concrete example of the decision logic an agent might follow:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> subprocess | |
| <span class="hljs-keyword">import</span> trackio | |
| api = trackio.Api() | |
| <span class="hljs-comment"># Read alerts from the last run</span> | |
| alerts = api.alerts(<span class="hljs-string">"cifar10-hparam-search"</span>, run=<span class="hljs-string">"run-lr0.001-bs64"</span>) | |
| <span class="hljs-comment"># Read the config from the last run</span> | |
| runs = api.runs(<span class="hljs-string">"cifar10-hparam-search"</span>) | |
| last_run = [r <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> runs <span class="hljs-keyword">if</span> r.name == <span class="hljs-string">"run-lr0.001-bs64"</span>][<span class="hljs-number">0</span>] | |
| config = last_run.config | |
| <span class="hljs-comment"># Decision logic based on alerts</span> | |
| new_config = <span class="hljs-built_in">dict</span>(config) | |
| <span class="hljs-keyword">for</span> alert <span class="hljs-keyword">in</span> alerts: | |
| <span class="hljs-keyword">if</span> <span class="hljs-string">"diverged"</span> <span class="hljs-keyword">in</span> alert[<span class="hljs-string">"title"</span>].lower(): | |
| new_config[<span class="hljs-string">"learning_rate"</span>] = config[<span class="hljs-string">"learning_rate"</span>] * <span class="hljs-number">0.1</span> | |
| <span class="hljs-keyword">elif</span> <span class="hljs-string">"overfitting"</span> <span class="hljs-keyword">in</span> alert[<span class="hljs-string">"title"</span>].lower(): | |
| new_config[<span class="hljs-string">"weight_decay"</span>] = config.get(<span class="hljs-string">"weight_decay"</span>, <span class="hljs-number">1e-4</span>) * <span class="hljs-number">10</span> | |
| <span class="hljs-keyword">elif</span> <span class="hljs-string">"early stopping"</span> <span class="hljs-keyword">in</span> alert[<span class="hljs-string">"title"</span>].lower(): | |
| new_config[<span class="hljs-string">"learning_rate"</span>] = config[<span class="hljs-string">"learning_rate"</span>] * <span class="hljs-number">0.5</span> | |
| <span class="hljs-keyword">elif</span> <span class="hljs-string">"high accuracy"</span> <span class="hljs-keyword">in</span> alert[<span class="hljs-string">"title"</span>].lower(): | |
| <span class="hljs-comment"># This config is good — try fine-tuning</span> | |
| new_config[<span class="hljs-string">"learning_rate"</span>] = config[<span class="hljs-string">"learning_rate"</span>] * <span class="hljs-number">0.5</span> | |
| new_config[<span class="hljs-string">"epochs"</span>] = config[<span class="hljs-string">"epochs"</span>] + <span class="hljs-number">20</span><!----></pre></div><!----> <p>The agent then generates a new training script with the updated config and launches it, repeating the loop.</p> <!--[1--><h2 class="relative group"><a id="comparing-runs" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#comparing-runs"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Comparing Runs</span></h2><!--]--><!----> <p>After several iterations, the agent can compare all runs in the project:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># List all runs</span> | |
| trackio list runs --project <span class="hljs-string">"cifar10-hparam-search"</span> --json | |
| <span class="hljs-comment"># Get a summary of each run</span> | |
| trackio get run --project <span class="hljs-string">"cifar10-hparam-search"</span> --run <span class="hljs-string">"run-lr0.001-bs64"</span> --json | |
| trackio get run --project <span class="hljs-string">"cifar10-hparam-search"</span> --run <span class="hljs-string">"run-lr0.0001-bs64"</span> --json<!----></pre></div><!----> <p>Or in Python:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->api = trackio.Api() | |
| runs = api.runs(<span class="hljs-string">"cifar10-hparam-search"</span>) | |
| <span class="hljs-keyword">for</span> run <span class="hljs-keyword">in</span> runs: | |
| alerts = run.alerts() | |
| errors = [a <span class="hljs-keyword">for</span> a <span class="hljs-keyword">in</span> alerts <span class="hljs-keyword">if</span> a[<span class="hljs-string">"level"</span>] == <span class="hljs-string">"error"</span>] | |
| warns = [a <span class="hljs-keyword">for</span> a <span class="hljs-keyword">in</span> alerts <span class="hljs-keyword">if</span> a[<span class="hljs-string">"level"</span>] == <span class="hljs-string">"warn"</span>] | |
| infos = [a <span class="hljs-keyword">for</span> a <span class="hljs-keyword">in</span> alerts <span class="hljs-keyword">if</span> a[<span class="hljs-string">"level"</span>] == <span class="hljs-string">"info"</span>] | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"<span class="hljs-subst">{run.name}</span>: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(errors)}</span> errors, <span class="hljs-subst">{<span class="hljs-built_in">len</span>(warns)}</span> warnings, <span class="hljs-subst">{<span class="hljs-built_in">len</span>(infos)}</span> info"</span>)<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="sending-alerts-to-the-agent-via-webhooks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#sending-alerts-to-the-agent-via-webhooks"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Sending Alerts to the Agent via Webhooks</span></h2><!--]--><!----> <p>If your agent framework has an HTTP endpoint or can receive messages from Slack/Discord, you can use webhooks to push alerts to the agent in real time instead of polling:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->trackio.init( | |
| project=<span class="hljs-string">"cifar10-hparam-search"</span>, | |
| webhook_url=<span class="hljs-string">"https://hooks.slack.com/services/T.../B.../xxx"</span>, | |
| )<!----></pre></div><!----> <p>This way, alerts are delivered to a Slack channel the agent monitors, closing the loop without any polling. See the <a href="alerts">Alerts guide</a> for detailed setup instructions for Slack and Discord.</p> <!--[1--><h2 class="relative group"><a id="full-example-autonomous-hyperparameter-search" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#full-example-autonomous-hyperparameter-search"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Full Example: Autonomous Hyperparameter Search</span></h2><!--]--><!----> <p>Here is a complete, self-contained example that ties everything together. This script runs a simple hyperparameter grid search with automated alert-based decisions:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> trackio | |
| <span class="hljs-keyword">import</span> random | |
| <span class="hljs-keyword">import</span> time | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">run_experiment</span>(<span class="hljs-params">config</span>): | |
| <span class="hljs-string">"""Simulate a training run. Replace with real training code."""</span> | |
| run = trackio.init( | |
| project=<span class="hljs-string">"auto-hparam-search"</span>, | |
| config=config, | |
| ) | |
| lr = config[<span class="hljs-string">"learning_rate"</span>] | |
| best_val_loss = <span class="hljs-built_in">float</span>(<span class="hljs-string">"inf"</span>) | |
| patience = <span class="hljs-number">0</span> | |
| <span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(config[<span class="hljs-string">"epochs"</span>]): | |
| <span class="hljs-comment"># Simulated metrics — replace with real training</span> | |
| noise = random.gauss(<span class="hljs-number">0</span>, lr * <span class="hljs-number">10</span>) | |
| train_loss = <span class="hljs-built_in">max</span>(<span class="hljs-number">0.01</span>, <span class="hljs-number">1.0</span> / (epoch + <span class="hljs-number">1</span>) + noise) | |
| val_loss = train_loss + random.uniform(<span class="hljs-number">0</span>, <span class="hljs-number">0.3</span>) | |
| val_acc = <span class="hljs-built_in">max</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">min</span>(<span class="hljs-number">1</span>, <span class="hljs-number">1.0</span> - val_loss + random.uniform(-<span class="hljs-number">0.05</span>, <span class="hljs-number">0.05</span>))) | |
| trackio.log({ | |
| <span class="hljs-string">"train_loss"</span>: train_loss, | |
| <span class="hljs-string">"val_loss"</span>: val_loss, | |
| <span class="hljs-string">"val_accuracy"</span>: val_acc, | |
| }) | |
| <span class="hljs-keyword">if</span> train_loss > <span class="hljs-number">5.0</span>: | |
| trackio.alert( | |
| title=<span class="hljs-string">"Training diverged"</span>, | |
| text=<span class="hljs-string">f"train_loss=<span class="hljs-subst">{train_loss:<span class="hljs-number">.4</span>f}</span>, lr=<span class="hljs-subst">{lr}</span>"</span>, | |
| level=trackio.AlertLevel.ERROR, | |
| ) | |
| <span class="hljs-keyword">break</span> | |
| <span class="hljs-keyword">if</span> val_loss < best_val_loss: | |
| best_val_loss = val_loss | |
| patience = <span class="hljs-number">0</span> | |
| <span class="hljs-keyword">else</span>: | |
| patience += <span class="hljs-number">1</span> | |
| <span class="hljs-keyword">if</span> patience >= <span class="hljs-number">5</span>: | |
| trackio.alert( | |
| title=<span class="hljs-string">"Early stopping"</span>, | |
| text=<span class="hljs-string">f"Best val_loss=<span class="hljs-subst">{best_val_loss:<span class="hljs-number">.4</span>f}</span>"</span>, | |
| level=trackio.AlertLevel.WARN, | |
| ) | |
| <span class="hljs-keyword">break</span> | |
| time.sleep(<span class="hljs-number">0.05</span>) | |
| trackio.alert( | |
| title=<span class="hljs-string">"Run complete"</span>, | |
| text=<span class="hljs-string">f"Final val_loss=<span class="hljs-subst">{val_loss:<span class="hljs-number">.4</span>f}</span>, val_acc=<span class="hljs-subst">{val_acc:<span class="hljs-number">.4</span>f}</span>, "</span> | |
| <span class="hljs-string">f"best_val_loss=<span class="hljs-subst">{best_val_loss:<span class="hljs-number">.4</span>f}</span>"</span>, | |
| level=trackio.AlertLevel.INFO, | |
| ) | |
| trackio.finish() | |
| <span class="hljs-keyword">return</span> run.name | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">analyze_and_decide</span>(<span class="hljs-params">api, project, run_name, config</span>): | |
| <span class="hljs-string">"""Read alerts from a run and suggest new hyperparameters."""</span> | |
| alerts = api.alerts(project, run=run_name) | |
| new_config = <span class="hljs-built_in">dict</span>(config) | |
| <span class="hljs-keyword">for</span> alert <span class="hljs-keyword">in</span> alerts: | |
| title = alert[<span class="hljs-string">"title"</span>].lower() | |
| <span class="hljs-keyword">if</span> <span class="hljs-string">"diverged"</span> <span class="hljs-keyword">in</span> title: | |
| new_config[<span class="hljs-string">"learning_rate"</span>] *= <span class="hljs-number">0.1</span> | |
| <span class="hljs-keyword">elif</span> <span class="hljs-string">"early stopping"</span> <span class="hljs-keyword">in</span> title: | |
| new_config[<span class="hljs-string">"learning_rate"</span>] *= <span class="hljs-number">0.5</span> | |
| <span class="hljs-keyword">elif</span> <span class="hljs-string">"run complete"</span> <span class="hljs-keyword">in</span> title <span class="hljs-keyword">and</span> <span class="hljs-string">"val_acc"</span> <span class="hljs-keyword">in</span> (alert.get(<span class="hljs-string">"text"</span>) <span class="hljs-keyword">or</span> <span class="hljs-string">""</span>): | |
| <span class="hljs-comment"># Slight refinement</span> | |
| new_config[<span class="hljs-string">"learning_rate"</span>] *= <span class="hljs-number">0.8</span> | |
| new_config[<span class="hljs-string">"epochs"</span>] += <span class="hljs-number">10</span> | |
| <span class="hljs-keyword">return</span> new_config | |
| api = trackio.Api() | |
| project = <span class="hljs-string">"auto-hparam-search"</span> | |
| config = { | |
| <span class="hljs-string">"learning_rate"</span>: <span class="hljs-number">0.01</span>, | |
| <span class="hljs-string">"batch_size"</span>: <span class="hljs-number">64</span>, | |
| <span class="hljs-string">"epochs"</span>: <span class="hljs-number">30</span>, | |
| } | |
| <span class="hljs-keyword">for</span> iteration <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>): | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"\n--- Iteration <span class="hljs-subst">{iteration + <span class="hljs-number">1</span>}</span> ---"</span>) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Config: <span class="hljs-subst">{config}</span>"</span>) | |
| run_name = run_experiment(config) | |
| config = analyze_and_decide(api, project, run_name, config)<!----></pre></div><!----> <p>This example is self-contained and uses simulated metrics. Replace the <code>run_experiment</code> function body with your real training code and the pattern stays the same.</p> <!--[1--><h2 class="relative group"><a id="using-alerts-with-transformers-and-trl" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-alerts-with-transformers-and-trl"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Using Alerts with Transformers and TRL</span></h2><!--]--><!----> <p>When using <code>report_to="trackio"</code> with the Hugging Face <code>Trainer</code>, the built-in <code>TrackioCallback</code> handles <code>trackio.init()</code>, <code>trackio.log()</code>, and <code>trackio.finish()</code> automatically. Since <code>trackio.init()</code> is called before training begins, the current run is set and <code>trackio.alert()</code> works from any other callback in the same process.</p> <p>To add alerts, write a small <code>TrainerCallback</code> and pass it to the Trainer via <code>callbacks=</code>:</p> <!--[2--><h3 class="relative group"><a id="transformers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Transformers</span></h3><!--]--><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> trackio | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainerCallback, TrainingArguments | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">AlertCallback</span>(<span class="hljs-title class_ inherited__">TrainerCallback</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">on_log</span>(<span class="hljs-params">self, args, state, control, logs=<span class="hljs-literal">None</span>, **kwargs</span>): | |
| <span class="hljs-keyword">if</span> <span class="hljs-string">"trackio"</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> args.report_to <span class="hljs-keyword">or</span> logs <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">return</span> | |
| loss = logs.get(<span class="hljs-string">"loss"</span>) | |
| <span class="hljs-keyword">if</span> loss <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">and</span> loss > <span class="hljs-number">5.0</span>: | |
| trackio.alert( | |
| title=<span class="hljs-string">"Training loss spike"</span>, | |
| text=<span class="hljs-string">f"loss=<span class="hljs-subst">{loss:<span class="hljs-number">.4</span>f}</span> at step <span class="hljs-subst">{state.global_step}</span>. "</span> | |
| <span class="hljs-string">"Consider lowering the learning rate."</span>, | |
| level=trackio.AlertLevel.ERROR, | |
| ) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">on_evaluate</span>(<span class="hljs-params">self, args, state, control, metrics=<span class="hljs-literal">None</span>, **kwargs</span>): | |
| <span class="hljs-keyword">if</span> <span class="hljs-string">"trackio"</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> args.report_to <span class="hljs-keyword">or</span> metrics <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">return</span> | |
| eval_loss = metrics.get(<span class="hljs-string">"eval_loss"</span>) | |
| <span class="hljs-keyword">if</span> eval_loss <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">and</span> eval_loss > <span class="hljs-number">2.0</span>: | |
| trackio.alert( | |
| title=<span class="hljs-string">"High eval loss"</span>, | |
| text=<span class="hljs-string">f"eval_loss=<span class="hljs-subst">{eval_loss:<span class="hljs-number">.4</span>f}</span> at step <span class="hljs-subst">{state.global_step}</span>."</span>, | |
| level=trackio.AlertLevel.WARN, | |
| ) | |
| trainer = Trainer( | |
| model=model, | |
| args=TrainingArguments( | |
| report_to=<span class="hljs-string">"trackio"</span>, | |
| project=<span class="hljs-string">"my-project"</span>, | |
| output_dir=<span class="hljs-string">"./output"</span>, | |
| num_train_epochs=<span class="hljs-number">3</span>, | |
| eval_strategy=<span class="hljs-string">"steps"</span>, | |
| eval_steps=<span class="hljs-number">100</span>, | |
| ), | |
| train_dataset=train_dataset, | |
| eval_dataset=eval_dataset, | |
| callbacks=[AlertCallback()], | |
| ) | |
| trainer.train()<!----></pre></div><!----> <p>An agent generating this code only needs to define the <code>AlertCallback</code> class with the right conditions, then add it to the <code>callbacks</code> list. The pattern is the same regardless of the model or dataset.</p> <!--[2--><h3 class="relative group"><a id="trl" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#trl"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>TRL</span></h3><!--]--><!----> <p>The same approach works with TRL trainers like <code>GRPOTrainer</code> or <code>SFTTrainer</code>. Here’s an example for reinforcement learning with alerts on reward collapse and KL divergence:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> trackio | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainerCallback | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">RLAlertCallback</span>(<span class="hljs-title class_ inherited__">TrainerCallback</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">on_log</span>(<span class="hljs-params">self, args, state, control, logs=<span class="hljs-literal">None</span>, **kwargs</span>): | |
| <span class="hljs-keyword">if</span> <span class="hljs-string">"trackio"</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> args.report_to <span class="hljs-keyword">or</span> logs <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">return</span> | |
| reward = logs.get(<span class="hljs-string">"train/reward"</span>) | |
| <span class="hljs-keyword">if</span> reward <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">and</span> reward < -<span class="hljs-number">1.0</span>: | |
| trackio.alert( | |
| title=<span class="hljs-string">"Reward collapse"</span>, | |
| text=<span class="hljs-string">f"reward=<span class="hljs-subst">{reward:<span class="hljs-number">.4</span>f}</span> at step <span class="hljs-subst">{state.global_step}</span>. "</span> | |
| <span class="hljs-string">"Consider lowering the learning rate or checking the "</span> | |
| <span class="hljs-string">"reward model."</span>, | |
| level=trackio.AlertLevel.ERROR, | |
| ) | |
| kl = logs.get(<span class="hljs-string">"train/kl"</span>) | |
| <span class="hljs-keyword">if</span> kl <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">and</span> kl > <span class="hljs-number">10.0</span>: | |
| trackio.alert( | |
| title=<span class="hljs-string">"KL divergence too high"</span>, | |
| text=<span class="hljs-string">f"kl=<span class="hljs-subst">{kl:<span class="hljs-number">.4</span>f}</span> at step <span class="hljs-subst">{state.global_step}</span>. "</span> | |
| <span class="hljs-string">"The policy is drifting too far from the reference model."</span>, | |
| level=trackio.AlertLevel.WARN, | |
| ) | |
| completion_length = logs.get(<span class="hljs-string">"train/completion_length"</span>) | |
| <span class="hljs-keyword">if</span> completion_length <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">and</span> completion_length < <span class="hljs-number">5.0</span>: | |
| trackio.alert( | |
| title=<span class="hljs-string">"Completions too short"</span>, | |
| text=<span class="hljs-string">f"Mean completion length=<span class="hljs-subst">{completion_length:<span class="hljs-number">.1</span>f}</span> tokens. "</span> | |
| <span class="hljs-string">"The model may be collapsing to short outputs."</span>, | |
| level=trackio.AlertLevel.WARN, | |
| )<!----></pre></div><!----> <p>Then pass it to any TRL trainer:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer, GRPOConfig | |
| trainer = GRPOTrainer( | |
| model=model, | |
| config=GRPOConfig( | |
| report_to=<span class="hljs-string">"trackio"</span>, | |
| project=<span class="hljs-string">"rl-experiment"</span>, | |
| output_dir=<span class="hljs-string">"./output"</span>, | |
| ), | |
| train_dataset=dataset, | |
| callbacks=[RLAlertCallback()], | |
| ) | |
| trainer.train()<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="tips-for-agents-writing-alert-callbacks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tips-for-agents-writing-alert-callbacks"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Tips for Agents Writing Alert Callbacks</span></h3><!--]--><!----> <ul><li><strong>Keep conditions simple.</strong> Each <code>if</code> block should check one metric against one threshold. This makes it easy for an agent to add, remove, or adjust conditions between runs.</li> <li><strong>Include the metric value and step in <code>text</code>.</strong> This gives the agent concrete numbers to reason about when deciding the next hyperparameters.</li> <li><strong>Include actionable suggestions.</strong> Text like “Consider lowering the learning rate” helps the agent decide what to change next.</li> <li><strong>Use <code>on_evaluate</code> for eval metrics.</strong> Eval metrics are only available in <code>on_evaluate</code>, not in <code>on_log</code>. Training metrics like <code>loss</code> appear in <code>on_log</code>.</li></ul> <!--[1--><h2 class="relative group"><a id="best-practices-summary" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#best-practices-summary"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Best Practices Summary</span></h2><!--]--><!----> <ol><li><p><strong>Use alerts as structured signals, not just notifications.</strong> Include numeric values and actionable suggestions in <code>text</code> so the agent can parse them and act on them.</p></li> <li><p><strong>Use severity levels consistently.</strong> <code>ERROR</code> = stop and change approach. <code>WARN</code> = tweak parameters. <code>INFO</code> = milestone reached.</p></li> <li><p><strong>Poll with <code>since</code> for efficiency.</strong> Save the timestamp of the last alert you processed and pass it as <code>since</code> on the next poll to avoid re-reading old alerts.</p></li> <li><p><strong>Keep all runs in the same project.</strong> This lets the agent compare runs side-by-side and see the full history of what was tried.</p></li> <li><p><strong>Log config with every run.</strong> Always pass <code>config=</code> to <code>trackio.init()</code> so the agent can read back which hyperparameters produced which results.</p></li> <li><p><strong>Use the CLI with <code>--json</code> for agent scripts.</strong> JSON output is reliable to parse and won’t break if the human-readable format changes.</p></li> <li><p><strong>Consider MCP for tight integration.</strong> If your agent framework supports MCP, running <code>trackio show --mcp-server</code> lets the agent call Trackio tools directly without spawning subprocesses.</p></li></ol> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/gradio-app/trackio/blob/main/docs/source/ml_agents.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_711gz4 = { | |
| base: "/docs/trackio/pr_672/en", | |
| assets: "/docs/trackio/pr_672/en" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/trackio/pr_672/en/_app/immutable/entry/start.B75uqjYQ.js"), | |
| import("/docs/trackio/pr_672/en/_app/immutable/entry/app.D_-Gc0EO.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 15], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 89 kB
- Xet hash:
- 14af2f70f3347924b7b3da367edd86485d21c0be78c5f9e117c089d3e823dcad
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.