abidlabs HF Staff commited on
Commit
fd6155d
·
verified ·
1 Parent(s): 3bad24b

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. trackio/CHANGELOG.md +232 -0
  3. trackio/__init__.py +755 -0
  4. trackio/_vendor/__init__.py +0 -0
  5. trackio/_vendor/gradio_exceptions.py +6 -0
  6. trackio/_vendor/networking.py +81 -0
  7. trackio/_vendor/tunneling.py +191 -0
  8. trackio/alerts.py +184 -0
  9. trackio/api.py +87 -0
  10. trackio/apple_gpu.py +253 -0
  11. trackio/asgi_app.py +232 -0
  12. trackio/assets/badge.png +3 -0
  13. trackio/assets/trackio_logo_dark.png +0 -0
  14. trackio/assets/trackio_logo_light.png +0 -0
  15. trackio/assets/trackio_logo_old.png +3 -0
  16. trackio/assets/trackio_logo_type_dark.png +0 -0
  17. trackio/assets/trackio_logo_type_dark_transparent.png +0 -0
  18. trackio/assets/trackio_logo_type_light.png +0 -0
  19. trackio/assets/trackio_logo_type_light_transparent.png +0 -0
  20. trackio/bucket_storage.py +152 -0
  21. trackio/cli.py +1272 -0
  22. trackio/cli_helpers.py +158 -0
  23. trackio/commit_scheduler.py +310 -0
  24. trackio/context_vars.py +18 -0
  25. trackio/deploy.py +1004 -0
  26. trackio/dummy_commit_scheduler.py +19 -0
  27. trackio/exceptions.py +2 -0
  28. trackio/frontend/dist/assets/index-CicmrdCP.css +1 -0
  29. trackio/frontend/dist/assets/index-D_4A8v8I.js +0 -0
  30. trackio/frontend/dist/index.html +14 -0
  31. trackio/frontend/eslint.config.js +42 -0
  32. trackio/frontend/index.html +13 -0
  33. trackio/frontend_server.py +64 -0
  34. trackio/gpu.py +357 -0
  35. trackio/histogram.py +71 -0
  36. trackio/imports.py +300 -0
  37. trackio/launch.py +202 -0
  38. trackio/launch_utils.py +33 -0
  39. trackio/markdown.py +21 -0
  40. trackio/mcp_setup.py +156 -0
  41. trackio/media/__init__.py +27 -0
  42. trackio/media/audio.py +167 -0
  43. trackio/media/image.py +84 -0
  44. trackio/media/media.py +79 -0
  45. trackio/media/utils.py +60 -0
  46. trackio/media/video.py +246 -0
  47. trackio/package.json +6 -0
  48. trackio/py.typed +0 -0
  49. trackio/remote_client.py +181 -0
  50. trackio/run.py +745 -0
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ trackio/assets/badge.png filter=lfs diff=lfs merge=lfs -text
37
+ trackio/assets/trackio_logo_old.png filter=lfs diff=lfs merge=lfs -text
trackio/CHANGELOG.md ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # trackio
2
+
3
+ ## 0.22.0
4
+
5
+ ### Features
6
+
7
+ - [#484](https://github.com/gradio-app/trackio/pull/484) [`cc05ada`](https://github.com/gradio-app/trackio/commit/cc05ada8e89773f3a894af99b801ef680f64418f) - Fix duplicate columns in parquet export. Thanks @abidlabs!
8
+ - [#487](https://github.com/gradio-app/trackio/pull/487) [`853f764`](https://github.com/gradio-app/trackio/commit/853f7646a70d12633afaa4f69db86425aa665413) - Relax `PIL` dependency and remove `plotly` as it's no longer used. Thanks @abidlabs!
9
+
10
+ ## 0.21.2
11
+
12
+ ### Features
13
+
14
+ - [#482](https://github.com/gradio-app/trackio/pull/482) [`f62180a`](https://github.com/gradio-app/trackio/commit/f62180a0218bc99a259d5ca110a0384a6cae11c8) - Use server-side bucket copy when freezing Spaces. Thanks @abidlabs!
15
+
16
+ ## 0.21.1
17
+
18
+ ### Features
19
+
20
+ - [#475](https://github.com/gradio-app/trackio/pull/475) [`fcb476c`](https://github.com/gradio-app/trackio/commit/fcb476cd37a40923e9679aaf966f41d582a878a8) - Tweaks. Thanks @abidlabs!
21
+ - [#477](https://github.com/gradio-app/trackio/pull/477) [`7d52dfd`](https://github.com/gradio-app/trackio/commit/7d52dfdce5b6eff6a34501a6d5a620220663cf09) - Fix `.sync()` and add `.freeze()` as a separate methods. Thanks @abidlabs!
22
+
23
+ ## 0.21.0
24
+
25
+ ### Features
26
+
27
+ - [#467](https://github.com/gradio-app/trackio/pull/467) [`f357deb`](https://github.com/gradio-app/trackio/commit/f357debf78957e4c1f2b901bee4f77cf397298b4) - Allow logged metrics as x-axis choices. Thanks @abidlabs!
28
+ - [#474](https://github.com/gradio-app/trackio/pull/474) [`655673d`](https://github.com/gradio-app/trackio/commit/655673d4c6b7c8b7ee8f87f2589f2dbbc3d2ef91) - Fix file descriptor leak from `sqlite3.connect`. Thanks @abidlabs!
29
+ - [#470](https://github.com/gradio-app/trackio/pull/470) [`bea8c9d`](https://github.com/gradio-app/trackio/commit/bea8c9dcae0b59d071b6c779c97ee525c9bbf6e7) - Restores tooltips to line plots and fixes the call to uses TTL instead of OAuth. Thanks @abidlabs!
30
+ - [#471](https://github.com/gradio-app/trackio/pull/471) [`246fce0`](https://github.com/gradio-app/trackio/commit/246fce0a01619e1c2c538c67b3e460883334d500) - Deprecate dataset backend in favor of buckets. Thanks @abidlabs!
31
+ - [#465](https://github.com/gradio-app/trackio/pull/465) [`3e11174`](https://github.com/gradio-app/trackio/commit/3e1117438bb8168b802245a33059affa558ae519) - Use HF buckets as backend. Thanks @abidlabs!
32
+ - [#469](https://github.com/gradio-app/trackio/pull/469) [`915d170`](https://github.com/gradio-app/trackio/commit/915d17045133172b59195acfdcc70709229668aa) - Make static Spaces work with Buckets and also allow conversion from Gradio SDK to Static Spaces. Thanks @abidlabs!
33
+
34
+ ## 0.20.2
35
+
36
+ ### Features
37
+
38
+ - [#464](https://github.com/gradio-app/trackio/pull/464) [`c89ebb3`](https://github.com/gradio-app/trackio/commit/c89ebb3b50f695bc7f16cbc6f46dce86f79a01e9) - Improve rendering of curves. Thanks @abidlabs!
39
+ - [#462](https://github.com/gradio-app/trackio/pull/462) [`9160b78`](https://github.com/gradio-app/trackio/commit/9160b78ff6f258f0b87a4f34a24e7d0b5dfbf2fb) - Refactor plot title to display only the metric name without the path. Thanks @qgallouedec!
40
+
41
+ ## 0.20.1
42
+
43
+ ### Features
44
+
45
+ - [#454](https://github.com/gradio-app/trackio/pull/454) [`22881db`](https://github.com/gradio-app/trackio/commit/22881dbbbb6b81197a00a19853771007093d61e4) - Bar chart single point. Thanks @abidlabs!
46
+ - [#455](https://github.com/gradio-app/trackio/pull/455) [`f8db51a`](https://github.com/gradio-app/trackio/commit/f8db51a20ca61ef703f3f2c2ee1ebd9c4f239cf2) - Adds a static Trackio mode via `trackio.sync(sdk="static")` and support for the `TRACKIO_SPACE_ID` environment variable. Thanks @abidlabs!
47
+
48
+ ## 0.20.0
49
+
50
+ ### Features
51
+
52
+ - [#450](https://github.com/gradio-app/trackio/pull/450) [`b0571ef`](https://github.com/gradio-app/trackio/commit/b0571ef6207a1ce346696f858ad2b7b584dd194f) - Use Svelte source for Gradio components directly in Trackio dashboard. Thanks @abidlabs!
53
+
54
+ ## 0.19.0
55
+
56
+ ### Features
57
+
58
+ - [#445](https://github.com/gradio-app/trackio/pull/445) [`cef4a58`](https://github.com/gradio-app/trackio/commit/cef4a583cb76f4091fc6c0e5783124ee84f8e243) - Add remote HF Space support to CLI. Thanks @abidlabs!
59
+ - [#444](https://github.com/gradio-app/trackio/pull/444) [`358f2a9`](https://github.com/gradio-app/trackio/commit/358f2a9ca238ee8b90b5a8c96220da287e0698fb) - Fix alerts placeholder flashing on reports page. Thanks @abidlabs!
60
+
61
+ ## 0.18.0
62
+
63
+ ### Features
64
+
65
+ - [#435](https://github.com/gradio-app/trackio/pull/435) [`4a47112`](https://github.com/gradio-app/trackio/commit/4a471128e18a39e45fad48a67fd711c5ae9e4aed) - feat: allow hiding section header accordions. Thanks @Saba9!
66
+ - [#439](https://github.com/gradio-app/trackio/pull/439) [`18e9650`](https://github.com/gradio-app/trackio/commit/18e96503d5a3a7cf926e92782d457e23c19942bd) - Add alerts with webhooks, CLI, and documentation. Thanks @abidlabs!
67
+ - [#438](https://github.com/gradio-app/trackio/pull/438) [`0875ccd`](https://github.com/gradio-app/trackio/commit/0875ccd3d8a41b1376f64030f21cfe8cdcc73b05) - Add "share this view" functionality. Thanks @qgallouedec!
68
+ - [#409](https://github.com/gradio-app/trackio/pull/409) [`9282403`](https://github.com/gradio-app/trackio/commit/9282403d8896d48679b0f888208a7ba5bdd4271a) - Add Apple Silicon GPU and system monitoring support. Thanks @znation!
69
+ - [#434](https://github.com/gradio-app/trackio/pull/434) [`4193223`](https://github.com/gradio-app/trackio/commit/41932230a3a2e1c16405dba08ecba5a42f11d1a8) - fix: table slider crash. Thanks @Saba9!
70
+
71
+ ### Fixes
72
+
73
+ - [#441](https://github.com/gradio-app/trackio/pull/441) [`3a2d11d`](https://github.com/gradio-app/trackio/commit/3a2d11dab0b4b37c925abc30ef84b0e2910321ee) - preserve x-axis step when toggling run checkboxes. Thanks @Saba9!
74
+
75
+ ## 0.17.0
76
+
77
+ ### Features
78
+
79
+ - [#428](https://github.com/gradio-app/trackio/pull/428) [`f7dd1ce`](https://github.com/gradio-app/trackio/commit/f7dd1ce2dc8a1936f9983467fcbcf93bfef01e09) - feat: add ability to rename runs. Thanks @Saba9!
80
+ - [#437](https://github.com/gradio-app/trackio/pull/437) [`2727c0b`](https://github.com/gradio-app/trackio/commit/2727c0b0755f48f7f186162ea45185c98f6b5516) - Add markdown reports across Trackio. Thanks @abidlabs!
81
+ - [#427](https://github.com/gradio-app/trackio/pull/427) [`5aeb9ed`](https://github.com/gradio-app/trackio/commit/5aeb9edcfd2068d309d9d64f172dcbcc327be1ab) - Make Trackio logging much more robust. Thanks @abidlabs!
82
+
83
+ ## 0.16.1
84
+
85
+ ### Features
86
+
87
+ - [#431](https://github.com/gradio-app/trackio/pull/431) [`c7ce55b`](https://github.com/gradio-app/trackio/commit/c7ce55b14dd5eb0c2165fb15df17dd60721c9325) - Lazy load the UI when trackio is imported. Thanks @abidlabs!
88
+
89
+ ## 0.16.0
90
+
91
+ ### Features
92
+
93
+ - [#426](https://github.com/gradio-app/trackio/pull/426) [`ead4dc8`](https://github.com/gradio-app/trackio/commit/ead4dc8e74ee2d8e47d61bca0a7668456acf49be) - Fix redundant double rendering of group checkboxes. Thanks @abidlabs!
94
+ - [#413](https://github.com/gradio-app/trackio/pull/413) [`39c4750`](https://github.com/gradio-app/trackio/commit/39c4750951d554ba6eb4d58847c6bb444b2891a8) - Check `dist-packages` when checking for source installation. Thanks @sergiopaniego!
95
+ - [#423](https://github.com/gradio-app/trackio/pull/423) [`2e52ab3`](https://github.com/gradio-app/trackio/commit/2e52ab303e3041718a6a56fbf84d0848aca9ad67) - Fix legend outline visibility issue. Thanks @Raghunath-Balaji!
96
+ - [#407](https://github.com/gradio-app/trackio/pull/407) [`c8a384d`](https://github.com/gradio-app/trackio/commit/c8a384ddfe5a295cecf862a26178d40e48acb424) - Fix pytests that were failling locally on MacOS. Thanks @abidlabs!
97
+ - [#405](https://github.com/gradio-app/trackio/pull/405) [`35aae4e`](https://github.com/gradio-app/trackio/commit/35aae4e3aa3e2b2888887528478b9dc6a9808bda) - Add conditional padding for HF Space dashboard when not in iframe. Thanks @znation!
98
+
99
+ ## 0.15.0
100
+
101
+ ### Features
102
+
103
+ - [#397](https://github.com/gradio-app/trackio/pull/397) [`6b38ad0`](https://github.com/gradio-app/trackio/commit/6b38ad02e5d73a0df49c4eede7e91331282ece04) - Adds `--host` cli option support. Thanks @abidlabs!
104
+ - [#396](https://github.com/gradio-app/trackio/pull/396) [`4a4d1ab`](https://github.com/gradio-app/trackio/commit/4a4d1ab85e63d923132a3fa7afa5d90e16431bec) - Fix run selection issue. Thanks @abidlabs!
105
+ - [#394](https://github.com/gradio-app/trackio/pull/394) [`c47a3a3`](https://github.com/gradio-app/trackio/commit/c47a3a31f8c4b83bce1aa7fc22eeba3d9021ad3d) - Add wandb-compatible API for trackio. Thanks @abidlabs!
106
+ - [#378](https://github.com/gradio-app/trackio/pull/378) [`b02046a`](https://github.com/gradio-app/trackio/commit/b02046a5b0dad7c9854e099a87f884afba4aecb2) - Add JSON export button for line plots and upgrade gradio dependency. Thanks @JamshedAli18!
107
+
108
+ ## 0.14.2
109
+
110
+ ### Features
111
+
112
+ - [#386](https://github.com/gradio-app/trackio/pull/386) [`f9452cd`](https://github.com/gradio-app/trackio/commit/f9452cdb8f0819368f3610f7ac0ed08957305275) - Fixing some issues related to deployed Trackio Spaces. Thanks @abidlabs!
113
+
114
+ ## 0.14.1
115
+
116
+ ### Features
117
+
118
+ - [#382](https://github.com/gradio-app/trackio/pull/382) [`44fe9bb`](https://github.com/gradio-app/trackio/commit/44fe9bb264fb2aafb0ec302ff15227c045819a2c) - Fix app file path when Trackio is not installed from source. Thanks @abidlabs!
119
+ - [#380](https://github.com/gradio-app/trackio/pull/380) [`c3f4cff`](https://github.com/gradio-app/trackio/commit/c3f4cff74bc5676e812773d8571454894fcdc7cc) - Add CLI commands for querying projects, runs, and metrics. Thanks @abidlabs!
120
+
121
+ ## 0.14.0
122
+
123
+ ### Features
124
+
125
+ - [#377](https://github.com/gradio-app/trackio/pull/377) [`5c5015b`](https://github.com/gradio-app/trackio/commit/5c5015b68c85c5de51111dad983f735c27b9a05f) - fixed wrapping issue in Runs table. Thanks @gaganchapa!
126
+ - [#374](https://github.com/gradio-app/trackio/pull/374) [`388e26b`](https://github.com/gradio-app/trackio/commit/388e26b9e9f24cd7ad203affe9b709be885b3d24) - Save Optimized Parquet files. Thanks @lhoestq!
127
+ - [#371](https://github.com/gradio-app/trackio/pull/371) [`fbace9c`](https://github.com/gradio-app/trackio/commit/fbace9cd7732c166f34d268f54b05bb06846cc5d) - Add GPU metrics logging. Thanks @kashif!
128
+ - [#367](https://github.com/gradio-app/trackio/pull/367) [`862840c`](https://github.com/gradio-app/trackio/commit/862840c13e30fc960cbee5b9eac4d3c25beba9de) - Add option to only show latest run, and fix the double logo issue. Thanks @abidlabs!
129
+
130
+ ## 0.13.1
131
+
132
+ ### Features
133
+
134
+ - [#369](https://github.com/gradio-app/trackio/pull/369) [`767e9fe`](https://github.com/gradio-app/trackio/commit/767e9fe095d7c6ed102016caf927c1517fb8618c) - tiny pr removing unnecessary code. Thanks @abidlabs!
135
+
136
+ ## 0.13.0
137
+
138
+ ### Features
139
+
140
+ - [#358](https://github.com/gradio-app/trackio/pull/358) [`073715d`](https://github.com/gradio-app/trackio/commit/073715d1caf8282f68890117f09c3ac301205312) - Improvements to `trackio.sync()`. Thanks @abidlabs!
141
+
142
+ ## 0.12.0
143
+
144
+ ### Features
145
+
146
+ - [#357](https://github.com/gradio-app/trackio/pull/357) [`02ba815`](https://github.com/gradio-app/trackio/commit/02ba815358060f1966052de051a5bdb09702920e) - Redesign media and tables to show up on separate page. Thanks @abidlabs!
147
+ - [#359](https://github.com/gradio-app/trackio/pull/359) [`08fe9c9`](https://github.com/gradio-app/trackio/commit/08fe9c9ddd7fe99ee811555fdfb62df9ab88e939) - docs: Improve docstrings. Thanks @qgallouedec!
148
+
149
+ ## 0.11.0
150
+
151
+ ### Features
152
+
153
+ - [#355](https://github.com/gradio-app/trackio/pull/355) [`ea51f49`](https://github.com/gradio-app/trackio/commit/ea51f4954922f21be76ef828700420fe9a912c4b) - Color code run checkboxes and match with plot lines. Thanks @abidlabs!
154
+ - [#353](https://github.com/gradio-app/trackio/pull/353) [`8abe691`](https://github.com/gradio-app/trackio/commit/8abe6919aeefe21fc7a23af814883efbb037c21f) - Remove show_api from demo.launch. Thanks @sergiopaniego!
155
+ - [#351](https://github.com/gradio-app/trackio/pull/351) [`8a8957e`](https://github.com/gradio-app/trackio/commit/8a8957e530dd7908d1fef7f2df030303f808101f) - Add `trackio.save()`. Thanks @abidlabs!
156
+
157
+ ## 0.10.0
158
+
159
+ ### Features
160
+
161
+ - [#305](https://github.com/gradio-app/trackio/pull/305) [`e64883a`](https://github.com/gradio-app/trackio/commit/e64883a51f7b8b93f7d48b8afe55acdb62238b71) - bump to gradio 6.0, make `trackio` compatible, and fix related issues. Thanks @abidlabs!
162
+
163
+ ## 0.9.1
164
+
165
+ ### Features
166
+
167
+ - [#344](https://github.com/gradio-app/trackio/pull/344) [`7e01024`](https://github.com/gradio-app/trackio/commit/7e010241d9a34794e0ce0dc19c1a6f0cf94ba856) - Avoid redundant calls to /whoami-v2. Thanks @Wauplin!
168
+
169
+ ## 0.9.0
170
+
171
+ ### Features
172
+
173
+ - [#343](https://github.com/gradio-app/trackio/pull/343) [`51bea30`](https://github.com/gradio-app/trackio/commit/51bea30f2877adff8e6497466d3a799400a0a049) - Sync offline projects to Hugging Face spaces. Thanks @candemircan!
174
+ - [#341](https://github.com/gradio-app/trackio/pull/341) [`4fd841f`](https://github.com/gradio-app/trackio/commit/4fd841fa190e15071b02f6fba7683ef4f393a654) - Adds a basic UI test to `trackio`. Thanks @abidlabs!
175
+ - [#339](https://github.com/gradio-app/trackio/pull/339) [`011d91b`](https://github.com/gradio-app/trackio/commit/011d91bb6ae266516fd250a349285670a8049d05) - Allow customzing the trackio color palette. Thanks @abidlabs!
176
+
177
+ ## 0.8.1
178
+
179
+ ### Features
180
+
181
+ - [#336](https://github.com/gradio-app/trackio/pull/336) [`5f9f51d`](https://github.com/gradio-app/trackio/commit/5f9f51dac8677f240d7c42c3e3b2660a22aee138) - Support a list of `Trackio.Image` in a `trackio.Table` cell. Thanks @abidlabs!
182
+
183
+ ## 0.8.0
184
+
185
+ ### Features
186
+
187
+ - [#331](https://github.com/gradio-app/trackio/pull/331) [`2c02d0f`](https://github.com/gradio-app/trackio/commit/2c02d0fd0a5824160528782402bb0dd4083396d5) - Truncate table string values that are greater than 250 characters (configuirable via env variable). Thanks @abidlabs!
188
+ - [#324](https://github.com/gradio-app/trackio/pull/324) [`50b2122`](https://github.com/gradio-app/trackio/commit/50b2122e7965ac82a72e6cb3b7d048bc10a2a6b1) - Add log y-axis functionality to UI. Thanks @abidlabs!
189
+ - [#326](https://github.com/gradio-app/trackio/pull/326) [`61dc1f4`](https://github.com/gradio-app/trackio/commit/61dc1f40af2f545f8e70395ddf0dbb8aee6b60d5) - Fix: improve table rendering for metrics in Trackio Dashboard. Thanks @vigneshwaran!
190
+ - [#328](https://github.com/gradio-app/trackio/pull/328) [`6857cbb`](https://github.com/gradio-app/trackio/commit/6857cbbe557a59a4642f210ec42566d108294e63) - Support trackio.Table with trackio.Image columns. Thanks @abidlabs!
191
+ - [#323](https://github.com/gradio-app/trackio/pull/323) [`6857cbb`](https://github.com/gradio-app/trackio/commit/6857cbbe557a59a4642f210ec42566d108294e63) - add Trackio client implementations in Go, Rust, and JS. Thanks @vaibhav-research!
192
+
193
+ ## 0.7.0
194
+
195
+ ### Features
196
+
197
+ - [#277](https://github.com/gradio-app/trackio/pull/277) [`db35601`](https://github.com/gradio-app/trackio/commit/db35601b9c023423c4654c9909b8ab73e58737de) - fix: make grouped runs view reflect live updates. Thanks @Saba9!
198
+ - [#320](https://github.com/gradio-app/trackio/pull/320) [`24ae739`](https://github.com/gradio-app/trackio/commit/24ae73969b09fb3126acd2f91647cdfbf8cf72a1) - Add additional query parms for xmin, xmax, and smoothing. Thanks @abidlabs!
199
+ - [#270](https://github.com/gradio-app/trackio/pull/270) [`cd1dfc3`](https://github.com/gradio-app/trackio/commit/cd1dfc3dc641b4499ac6d4a1b066fa8e2b52c57b) - feature: add support for logging audio. Thanks @Saba9!
200
+
201
+ ## 0.6.0
202
+
203
+ ### Features
204
+
205
+ - [#309](https://github.com/gradio-app/trackio/pull/309) [`1df2353`](https://github.com/gradio-app/trackio/commit/1df23534d6c01938c8db9c0f584ffa23e8d6021d) - Add histogram support with wandb-compatible API. Thanks @abidlabs!
206
+ - [#315](https://github.com/gradio-app/trackio/pull/315) [`76ba060`](https://github.com/gradio-app/trackio/commit/76ba06055dc43ca8f03b79f3e72d761949bd19a8) - Add guards to avoid silent fails. Thanks @Xmaster6y!
207
+ - [#313](https://github.com/gradio-app/trackio/pull/313) [`a606b3e`](https://github.com/gradio-app/trackio/commit/a606b3e1c5edf3d4cf9f31bd50605226a5a1c5d0) - No longer prevent certain keys from being used. Instead, dunderify them to prevent collisions with internal usage. Thanks @abidlabs!
208
+ - [#317](https://github.com/gradio-app/trackio/pull/317) [`27370a5`](https://github.com/gradio-app/trackio/commit/27370a595d0dbdf7eebbe7159d2ba778f039da44) - quick fixes for trackio.histogram. Thanks @abidlabs!
209
+ - [#312](https://github.com/gradio-app/trackio/pull/312) [`aa0f3bf`](https://github.com/gradio-app/trackio/commit/aa0f3bf372e7a0dd592a38af699c998363830eeb) - Fix video logging by adding TRACKIO_DIR to allowed_paths. Thanks @abidlabs!
210
+
211
+ ## 0.5.3
212
+
213
+ ### Features
214
+
215
+ - [#300](https://github.com/gradio-app/trackio/pull/300) [`5e4cacf`](https://github.com/gradio-app/trackio/commit/5e4cacf2e7ce527b4ce60de3a5bc05d2c02c77fb) - Adds more environment variables to allow customization of Trackio dashboard. Thanks @abidlabs!
216
+
217
+ ## 0.5.2
218
+
219
+ ### Features
220
+
221
+ - [#293](https://github.com/gradio-app/trackio/pull/293) [`64afc28`](https://github.com/gradio-app/trackio/commit/64afc28d3ea1dfd821472dc6bf0b8ed35a9b74be) - Ensures that the TRACKIO_DIR environment variable is respected. Thanks @abidlabs!
222
+ - [#287](https://github.com/gradio-app/trackio/pull/287) [`cd3e929`](https://github.com/gradio-app/trackio/commit/cd3e9294320949e6b8b829239069a43d5d7ff4c1) - fix(sqlite): unify .sqlite extension, allow export when DBs exist, clean WAL sidecars on import. Thanks @vaibhav-research!
223
+
224
+ ### Fixes
225
+
226
+ - [#291](https://github.com/gradio-app/trackio/pull/291) [`3b5adc3`](https://github.com/gradio-app/trackio/commit/3b5adc3d1f452dbab7a714d235f4974782f93730) - Fix the wheel build. Thanks @pngwn!
227
+
228
+ ## 0.5.1
229
+
230
+ ### Fixes
231
+
232
+ - [#278](https://github.com/gradio-app/trackio/pull/278) [`314c054`](https://github.com/gradio-app/trackio/commit/314c05438007ddfea3383e06fd19143e27468e2d) - Fix row orientation of metrics plots. Thanks @abidlabs!
trackio/__init__.py ADDED
@@ -0,0 +1,755 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import atexit
2
+ import glob
3
+ import json
4
+ import logging
5
+ import os
6
+ import shutil
7
+ import warnings
8
+ import webbrowser
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import huggingface_hub
13
+ from gradio_client import handle_file
14
+ from huggingface_hub import SpaceStorage
15
+ from huggingface_hub.errors import LocalTokenNotFoundError
16
+
17
+ from trackio import context_vars, deploy, utils
18
+ from trackio.alerts import AlertLevel
19
+ from trackio.api import Api
20
+ from trackio.apple_gpu import apple_gpu_available
21
+ from trackio.apple_gpu import log_apple_gpu as _log_apple_gpu
22
+ from trackio.deploy import freeze, sync
23
+ from trackio.gpu import gpu_available
24
+ from trackio.gpu import log_gpu as _log_nvidia_gpu
25
+ from trackio.histogram import Histogram
26
+ from trackio.imports import import_csv, import_tf_events
27
+ from trackio.launch import launch_trackio_dashboard
28
+ from trackio.markdown import Markdown
29
+ from trackio.media import (
30
+ TrackioAudio,
31
+ TrackioImage,
32
+ TrackioVideo,
33
+ get_project_media_path,
34
+ )
35
+ from trackio.remote_client import RemoteClient
36
+ from trackio.run import Run
37
+ from trackio.server import TrackioDashboardApp, build_starlette_app_only
38
+ from trackio.sqlite_storage import SQLiteStorage
39
+ from trackio.table import Table
40
+ from trackio.typehints import UploadEntry
41
+ from trackio.utils import TRACKIO_DIR
42
+
43
+ logging.getLogger("httpx").setLevel(logging.WARNING)
44
+
45
+ __version__ = json.loads(Path(__file__).parent.joinpath("package.json").read_text())[
46
+ "version"
47
+ ]
48
+
49
+
50
+ class _TupleNoPrint(tuple):
51
+ def __repr__(self) -> str:
52
+ return ""
53
+
54
+
55
+ __all__ = [
56
+ "init",
57
+ "log",
58
+ "log_system",
59
+ "log_gpu",
60
+ "finish",
61
+ "alert",
62
+ "AlertLevel",
63
+ "show",
64
+ "sync",
65
+ "freeze",
66
+ "delete_project",
67
+ "import_csv",
68
+ "import_tf_events",
69
+ "save",
70
+ "Image",
71
+ "Video",
72
+ "Audio",
73
+ "Table",
74
+ "Histogram",
75
+ "Markdown",
76
+ "Api",
77
+ ]
78
+
79
+ Audio = TrackioAudio
80
+ Image = TrackioImage
81
+ Video = TrackioVideo
82
+
83
+
84
+ config = {}
85
+
86
+ _atexit_registered = False
87
+ _projects_notified_auto_log_hw: set[str] = set()
88
+
89
+
90
+ def _cleanup_current_run():
91
+ run = context_vars.current_run.get()
92
+ if run is not None:
93
+ try:
94
+ run.finish()
95
+ except Exception:
96
+ pass
97
+
98
+
99
+ def init(
100
+ project: str,
101
+ name: str | None = None,
102
+ group: str | None = None,
103
+ space_id: str | None = None,
104
+ space_storage: SpaceStorage | None = None,
105
+ dataset_id: str | None = None,
106
+ bucket_id: str | None = None,
107
+ config: dict | None = None,
108
+ resume: str = "never",
109
+ settings: Any = None,
110
+ private: bool | None = None,
111
+ embed: bool = True,
112
+ auto_log_gpu: bool | None = None,
113
+ gpu_log_interval: float = 10.0,
114
+ webhook_url: str | None = None,
115
+ webhook_min_level: AlertLevel | str | None = None,
116
+ ) -> Run:
117
+ """
118
+ Creates a new Trackio project and returns a [`Run`] object.
119
+
120
+ Args:
121
+ project (`str`):
122
+ The name of the project (can be an existing project to continue tracking or
123
+ a new project to start tracking from scratch).
124
+ name (`str`, *optional*):
125
+ The name of the run (if not provided, a default name will be generated).
126
+ group (`str`, *optional*):
127
+ The name of the group which this run belongs to in order to help organize
128
+ related runs together. You can toggle the entire group's visibilitiy in the
129
+ dashboard.
130
+ space_id (`str`, *optional*):
131
+ If provided, the project will be logged to a Hugging Face Space instead of
132
+ a local directory. Should be a complete Space name like
133
+ `"username/reponame"` or `"orgname/reponame"`, or just `"reponame"` in which
134
+ case the Space will be created in the currently-logged-in Hugging Face
135
+ user's namespace. If the Space does not exist, it will be created. If the
136
+ Space already exists, the project will be logged to it. Can also be set
137
+ via the `TRACKIO_SPACE_ID` environment variable. You cannot log to a
138
+ Space that has been **frozen** (converted to the static SDK); use
139
+ ``trackio.sync(..., sdk="static")`` only after you are done logging.
140
+ space_storage ([`~huggingface_hub.SpaceStorage`], *optional*):
141
+ Choice of persistent storage tier.
142
+ dataset_id (`str`, *optional*):
143
+ Deprecated. Use `bucket_id` instead.
144
+ bucket_id (`str`, *optional*):
145
+ The ID of the Hugging Face Bucket to use for metric persistence. By default,
146
+ when a `space_id` is provided and `bucket_id` is not explicitly set, a
147
+ bucket is auto-generated from the space_id. Buckets provide
148
+ S3-like storage without git overhead - the SQLite database is stored directly
149
+ via `hf-mount` in the Space. Specify a Bucket with name like
150
+ `"username/bucketname"` or just `"bucketname"`.
151
+ config (`dict`, *optional*):
152
+ A dictionary of configuration options. Provided for compatibility with
153
+ `wandb.init()`.
154
+ resume (`str`, *optional*, defaults to `"never"`):
155
+ Controls how to handle resuming a run. Can be one of:
156
+
157
+ - `"must"`: Must resume the run with the given name, raises error if run
158
+ doesn't exist
159
+ - `"allow"`: Resume the run if it exists, otherwise create a new run
160
+ - `"never"`: Never resume a run, always create a new one
161
+ private (`bool`, *optional*):
162
+ Whether to make the Space private. If None (default), the repo will be
163
+ public unless the organization's default is private. This value is ignored
164
+ if the repo already exists.
165
+ settings (`Any`, *optional*):
166
+ Not used. Provided for compatibility with `wandb.init()`.
167
+ embed (`bool`, *optional*, defaults to `True`):
168
+ If running inside a Jupyter/Colab notebook, whether the dashboard should
169
+ automatically be embedded in the cell when trackio.init() is called. For
170
+ local runs, this launches a local Trackio dashboard and embeds it. For Space runs,
171
+ this embeds the Space URL. In Colab, the local dashboard will be accessible
172
+ via a public share URL when `share=True`.
173
+ auto_log_gpu (`bool` or `None`, *optional*, defaults to `None`):
174
+ Controls automatic GPU metrics logging. If `None` (default), GPU logging
175
+ is automatically enabled when `nvidia-ml-py` is installed and an NVIDIA
176
+ GPU or Apple M series is detected. Set to `True` to force enable or
177
+ `False` to disable.
178
+ gpu_log_interval (`float`, *optional*, defaults to `10.0`):
179
+ The interval in seconds between automatic GPU metric logs.
180
+ Only used when `auto_log_gpu=True`.
181
+ webhook_url (`str`, *optional*):
182
+ A webhook URL to POST alert payloads to when `trackio.alert()` is
183
+ called. Supports Slack and Discord webhook URLs natively (payloads
184
+ are formatted automatically). Can also be set via the
185
+ `TRACKIO_WEBHOOK_URL` environment variable. Individual alerts can
186
+ override this URL by passing `webhook_url` to `trackio.alert()`.
187
+ webhook_min_level (`AlertLevel` or `str`, *optional*):
188
+ Minimum alert level that should trigger webhook delivery.
189
+ For example, `AlertLevel.WARN` sends only `WARN` and `ERROR`
190
+ alerts to the webhook destination. Can also be set via
191
+ `TRACKIO_WEBHOOK_MIN_LEVEL`.
192
+ Returns:
193
+ `Run`: A [`Run`] object that can be used to log metrics and finish the run.
194
+ """
195
+ if settings is not None:
196
+ warnings.warn(
197
+ "* Warning: settings is not used. Provided for compatibility with wandb.init(). Please create an issue at: https://github.com/gradio-app/trackio/issues if you need a specific feature implemented."
198
+ )
199
+
200
+ space_id = space_id or os.environ.get("TRACKIO_SPACE_ID")
201
+ bucket_id = bucket_id or os.environ.get("TRACKIO_BUCKET_ID")
202
+ if space_id is None and dataset_id is not None:
203
+ raise ValueError("Must provide a `space_id` when `dataset_id` is provided.")
204
+ if dataset_id is not None and bucket_id is not None:
205
+ raise ValueError("Cannot provide both `dataset_id` and `bucket_id`.")
206
+ try:
207
+ space_id, dataset_id, bucket_id = utils.preprocess_space_and_dataset_ids(
208
+ space_id, dataset_id, bucket_id
209
+ )
210
+ except LocalTokenNotFoundError as e:
211
+ raise LocalTokenNotFoundError(
212
+ f"You must be logged in to Hugging Face locally when `space_id` is provided to deploy to a Space. {e}"
213
+ ) from e
214
+
215
+ if space_id is None and bucket_id is not None:
216
+ warnings.warn(
217
+ "trackio.init() has `bucket_id` set but `space_id` is None: metrics will be logged "
218
+ "locally only. Pass `space_id` to create or use a Hugging Face Space, which will be "
219
+ "attached to the Hugging Face Bucket.",
220
+ UserWarning,
221
+ stacklevel=2,
222
+ )
223
+
224
+ if space_id is not None:
225
+ deploy.raise_if_space_is_frozen_for_logging(space_id)
226
+
227
+ url = context_vars.current_server.get()
228
+
229
+ if space_id is not None:
230
+ if url is None:
231
+ url = space_id
232
+ context_vars.current_server.set(url)
233
+ context_vars.current_space_id.set(space_id)
234
+
235
+ _should_embed_local = False
236
+
237
+ if (
238
+ context_vars.current_project.get() is None
239
+ or context_vars.current_project.get() != project
240
+ ):
241
+ print(f"* Trackio project initialized: {project}")
242
+
243
+ if bucket_id is not None:
244
+ os.environ["TRACKIO_BUCKET_ID"] = bucket_id
245
+ bucket_url = f"https://huggingface.co/buckets/{bucket_id}"
246
+ print(
247
+ f"* Trackio metrics will be synced to Hugging Face Bucket: {bucket_url}"
248
+ )
249
+ elif dataset_id is not None:
250
+ os.environ["TRACKIO_DATASET_ID"] = dataset_id
251
+ print(
252
+ f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}"
253
+ )
254
+ if space_id is None:
255
+ print(f"* Trackio metrics logged to: {TRACKIO_DIR}")
256
+ _should_embed_local = embed and utils.is_in_notebook()
257
+ if not _should_embed_local:
258
+ utils.print_dashboard_instructions(project)
259
+ else:
260
+ deploy.create_space_if_not_exists(
261
+ space_id,
262
+ space_storage,
263
+ dataset_id,
264
+ bucket_id,
265
+ private,
266
+ )
267
+ user_name, space_name = space_id.split("/")
268
+ space_url = deploy.SPACE_HOST_URL.format(
269
+ user_name=user_name, space_name=space_name
270
+ )
271
+ if utils.is_in_notebook() and embed:
272
+ utils.embed_url_in_notebook(space_url)
273
+ context_vars.current_project.set(project)
274
+
275
+ if resume == "must":
276
+ if name is None:
277
+ raise ValueError("Must provide a run name when resume='must'")
278
+ if name not in SQLiteStorage.get_runs(project):
279
+ raise ValueError(f"Run '{name}' does not exist in project '{project}'")
280
+ resumed = True
281
+ elif resume == "allow":
282
+ resumed = name is not None and name in SQLiteStorage.get_runs(project)
283
+ elif resume == "never":
284
+ if name is not None and name in SQLiteStorage.get_runs(project):
285
+ warnings.warn(
286
+ f"* Warning: resume='never' but a run '{name}' already exists in "
287
+ f"project '{project}'. Generating a new name and instead. If you want "
288
+ "to resume this run, call init() with resume='must' or resume='allow'."
289
+ )
290
+ name = None
291
+ resumed = False
292
+ else:
293
+ raise ValueError("resume must be one of: 'must', 'allow', or 'never'")
294
+
295
+ if auto_log_gpu is None:
296
+ nvidia_available = gpu_available()
297
+ apple_available = apple_gpu_available()
298
+ auto_log_gpu = nvidia_available or apple_available
299
+ if project not in _projects_notified_auto_log_hw:
300
+ if nvidia_available:
301
+ print("* NVIDIA GPU detected, enabling automatic GPU metrics logging")
302
+ elif apple_available:
303
+ print(
304
+ "* Apple Silicon detected, enabling automatic system metrics logging"
305
+ )
306
+ if nvidia_available or apple_available:
307
+ _projects_notified_auto_log_hw.add(project)
308
+
309
+ run = Run(
310
+ url=url,
311
+ project=project,
312
+ client=None,
313
+ name=name,
314
+ group=group,
315
+ config=config,
316
+ space_id=space_id,
317
+ auto_log_gpu=auto_log_gpu,
318
+ gpu_log_interval=gpu_log_interval,
319
+ webhook_url=webhook_url,
320
+ webhook_min_level=webhook_min_level,
321
+ )
322
+
323
+ if space_id is not None:
324
+ SQLiteStorage.set_project_metadata(project, "space_id", space_id)
325
+ if SQLiteStorage.has_pending_data(project):
326
+ run._has_local_buffer = True
327
+
328
+ global _atexit_registered
329
+ if not _atexit_registered:
330
+ atexit.register(_cleanup_current_run)
331
+ _atexit_registered = True
332
+
333
+ if resumed:
334
+ print(f"* Resumed existing run: {run.name}")
335
+ else:
336
+ print(f"* Created new run: {run.name}")
337
+
338
+ context_vars.current_run.set(run)
339
+ globals()["config"] = run.config
340
+
341
+ if _should_embed_local:
342
+ show(project=project, open_browser=False, block_thread=False)
343
+
344
+ return run
345
+
346
+
347
+ def log(metrics: dict, step: int | None = None) -> None:
348
+ """
349
+ Logs metrics to the current run.
350
+
351
+ Args:
352
+ metrics (`dict`):
353
+ A dictionary of metrics to log.
354
+ step (`int`, *optional*):
355
+ The step number. If not provided, the step will be incremented
356
+ automatically.
357
+ """
358
+ run = context_vars.current_run.get()
359
+ if run is None:
360
+ raise RuntimeError("Call trackio.init() before trackio.log().")
361
+ run.log(
362
+ metrics=metrics,
363
+ step=step,
364
+ )
365
+
366
+
367
+ def log_system(metrics: dict) -> None:
368
+ """
369
+ Logs system metrics (GPU, etc.) to the current run using timestamps instead of steps.
370
+
371
+ Args:
372
+ metrics (`dict`):
373
+ A dictionary of system metrics to log.
374
+ """
375
+ run = context_vars.current_run.get()
376
+ if run is None:
377
+ raise RuntimeError("Call trackio.init() before trackio.log_system().")
378
+ run.log_system(metrics=metrics)
379
+
380
+
381
+ def log_gpu(run: Run | None = None, device: int | None = None) -> dict:
382
+ """
383
+ Log GPU metrics to the current or specified run as system metrics.
384
+ Automatically detects whether an NVIDIA or Apple GPU is available and calls
385
+ the appropriate logging method.
386
+
387
+ Args:
388
+ run: Optional Run instance. If None, uses current run from context.
389
+ device: CUDA device index to collect metrics from (NVIDIA GPUs only).
390
+ If None, collects from all GPUs visible to this process.
391
+ This parameter is ignored for Apple GPUs.
392
+
393
+ Returns:
394
+ dict: The GPU metrics that were logged.
395
+
396
+ Example:
397
+ ```python
398
+ import trackio
399
+
400
+ run = trackio.init(project="my-project")
401
+ trackio.log({"loss": 0.5})
402
+ trackio.log_gpu()
403
+ trackio.log_gpu(device=0)
404
+ ```
405
+ """
406
+ if run is None:
407
+ run = context_vars.current_run.get()
408
+ if run is None:
409
+ raise RuntimeError("Call trackio.init() before trackio.log_gpu().")
410
+
411
+ if gpu_available():
412
+ return _log_nvidia_gpu(run=run, device=device)
413
+ elif apple_gpu_available():
414
+ return _log_apple_gpu(run=run)
415
+ else:
416
+ warnings.warn(
417
+ "No GPU detected. Install nvidia-ml-py for NVIDIA GPU support "
418
+ "or psutil for Apple Silicon support."
419
+ )
420
+ return {}
421
+
422
+
423
+ def finish():
424
+ """
425
+ Finishes the current run.
426
+ """
427
+ run = context_vars.current_run.get()
428
+ if run is None:
429
+ raise RuntimeError("Call trackio.init() before trackio.finish().")
430
+ run.finish()
431
+
432
+
433
+ def alert(
434
+ title: str,
435
+ text: str | None = None,
436
+ level: AlertLevel = AlertLevel.WARN,
437
+ webhook_url: str | None = None,
438
+ ) -> None:
439
+ """
440
+ Fires an alert immediately on the current run. The alert is printed to the
441
+ terminal, stored in the database, and displayed in the dashboard. If a
442
+ webhook URL is configured (via `trackio.init()`, the `TRACKIO_WEBHOOK_URL`
443
+ environment variable, or the `webhook_url` parameter here), the alert is
444
+ also POSTed to that URL.
445
+
446
+ Args:
447
+ title (`str`):
448
+ A short title for the alert.
449
+ text (`str`, *optional*):
450
+ A longer description with details about the alert.
451
+ level (`AlertLevel`, *optional*, defaults to `AlertLevel.WARN`):
452
+ The severity level. One of `AlertLevel.INFO`, `AlertLevel.WARN`,
453
+ or `AlertLevel.ERROR`.
454
+ webhook_url (`str`, *optional*):
455
+ A webhook URL to send this specific alert to. Overrides any
456
+ URL set in `trackio.init()` or the `TRACKIO_WEBHOOK_URL`
457
+ environment variable. Supports Slack and Discord webhook
458
+ URLs natively.
459
+ """
460
+ run = context_vars.current_run.get()
461
+ if run is None:
462
+ raise RuntimeError("Call trackio.init() before trackio.alert().")
463
+ run.alert(title=title, text=text, level=level, webhook_url=webhook_url)
464
+
465
+
466
+ def delete_project(project: str, force: bool = False) -> bool:
467
+ """
468
+ Deletes a project by removing its local SQLite database.
469
+
470
+ Args:
471
+ project (`str`):
472
+ The name of the project to delete.
473
+ force (`bool`, *optional*, defaults to `False`):
474
+ If `True`, deletes the project without prompting for confirmation.
475
+ If `False`, prompts the user to confirm before deleting.
476
+
477
+ Returns:
478
+ `bool`: `True` if the project was deleted, `False` otherwise.
479
+ """
480
+ db_path = SQLiteStorage.get_project_db_path(project)
481
+
482
+ if not db_path.exists():
483
+ print(f"* Project '{project}' does not exist.")
484
+ return False
485
+
486
+ if not force:
487
+ response = input(
488
+ f"Are you sure you want to delete project '{project}'? "
489
+ f"This will permanently delete all runs and metrics. (y/N): "
490
+ )
491
+ if response.lower() not in ["y", "yes"]:
492
+ print("* Deletion cancelled.")
493
+ return False
494
+
495
+ try:
496
+ db_path.unlink()
497
+
498
+ for suffix in ("-wal", "-shm"):
499
+ sidecar = Path(str(db_path) + suffix)
500
+ if sidecar.exists():
501
+ sidecar.unlink()
502
+
503
+ print(f"* Project '{project}' has been deleted.")
504
+ return True
505
+ except Exception as e:
506
+ print(f"* Error deleting project '{project}': {e}")
507
+ return False
508
+
509
+
510
+ def save(
511
+ glob_str: str | Path,
512
+ project: str | None = None,
513
+ ) -> str:
514
+ """
515
+ Saves files to a project (not linked to a specific run). If Trackio is running
516
+ locally, the file(s) will be copied to the project's files directory. If Trackio is
517
+ running in a Space, the file(s) will be uploaded to the Space's files directory.
518
+
519
+ Args:
520
+ glob_str (`str` or `Path`):
521
+ The file path or glob pattern to save. Can be a single file or a pattern
522
+ matching multiple files (e.g., `"*.py"`, `"models/**/*.pth"`).
523
+ project (`str`, *optional*):
524
+ The name of the project to save files to. If not provided, uses the current
525
+ project from `trackio.init()`. If no project is initialized, raises an
526
+ error.
527
+
528
+ Returns:
529
+ `str`: The path where the file(s) were saved (project's files directory).
530
+
531
+ Example:
532
+ ```python
533
+ import trackio
534
+
535
+ trackio.init(project="my-project")
536
+ trackio.save("config.yaml")
537
+ trackio.save("models/*.pth")
538
+ ```
539
+ """
540
+ if project is None:
541
+ project = context_vars.current_project.get()
542
+ if project is None:
543
+ raise RuntimeError(
544
+ "No project specified. Either call trackio.init() first or provide a "
545
+ "project parameter to trackio.save()."
546
+ )
547
+
548
+ glob_str = Path(glob_str)
549
+ base_path = Path.cwd().resolve()
550
+
551
+ matched_files = []
552
+ if glob_str.is_file():
553
+ matched_files = [glob_str.resolve()]
554
+ else:
555
+ pattern = str(glob_str)
556
+ if not glob_str.is_absolute():
557
+ pattern = str((Path.cwd() / glob_str).resolve())
558
+ matched_files = [
559
+ Path(f).resolve()
560
+ for f in glob.glob(pattern, recursive=True)
561
+ if Path(f).is_file()
562
+ ]
563
+
564
+ if not matched_files:
565
+ raise ValueError(f"No files found matching pattern: {glob_str}")
566
+
567
+ current_run = context_vars.current_run.get()
568
+ is_local = (
569
+ current_run._is_local
570
+ if current_run is not None
571
+ else (context_vars.current_space_id.get() is None)
572
+ )
573
+
574
+ if is_local:
575
+ for file_path in matched_files:
576
+ try:
577
+ relative_to_base = file_path.relative_to(base_path)
578
+ except ValueError:
579
+ relative_to_base = Path(file_path.name)
580
+
581
+ if current_run is not None:
582
+ current_run._queue_upload(
583
+ file_path,
584
+ step=None,
585
+ relative_path=str(relative_to_base.parent),
586
+ use_run_name=False,
587
+ )
588
+ else:
589
+ media_path = get_project_media_path(
590
+ project=project,
591
+ run=None,
592
+ step=None,
593
+ relative_path=str(relative_to_base),
594
+ )
595
+ shutil.copy(str(file_path), str(media_path))
596
+ else:
597
+ url = context_vars.current_server.get()
598
+
599
+ upload_entries = []
600
+ for file_path in matched_files:
601
+ try:
602
+ relative_to_base = file_path.relative_to(base_path)
603
+ except ValueError:
604
+ relative_to_base = Path(file_path.name)
605
+
606
+ if current_run is not None:
607
+ current_run._queue_upload(
608
+ file_path,
609
+ step=None,
610
+ relative_path=str(relative_to_base.parent),
611
+ use_run_name=False,
612
+ )
613
+ else:
614
+ upload_entry: UploadEntry = {
615
+ "project": project,
616
+ "run": None,
617
+ "step": None,
618
+ "relative_path": str(relative_to_base),
619
+ "uploaded_file": handle_file(file_path),
620
+ }
621
+ upload_entries.append(upload_entry)
622
+
623
+ if upload_entries:
624
+ if url is None:
625
+ raise RuntimeError(
626
+ "No server available. Call trackio.init() before trackio.save() to start the server."
627
+ )
628
+
629
+ try:
630
+ client = RemoteClient(
631
+ url,
632
+ hf_token=huggingface_hub.utils.get_token(),
633
+ httpx_kwargs={"timeout": 90},
634
+ )
635
+ client.predict(
636
+ api_name="/bulk_upload_media",
637
+ uploads=upload_entries,
638
+ hf_token=huggingface_hub.utils.get_token(),
639
+ )
640
+ except Exception as e:
641
+ warnings.warn(
642
+ f"Failed to upload files: {e}. "
643
+ "Files may not be available in the dashboard."
644
+ )
645
+
646
+ return str(utils.MEDIA_DIR / project / "files")
647
+
648
+
649
+ def show(
650
+ project: str | None = None,
651
+ *,
652
+ theme: Any = None,
653
+ mcp_server: bool | None = None,
654
+ footer: bool = True,
655
+ color_palette: list[str] | None = None,
656
+ open_browser: bool = True,
657
+ block_thread: bool | None = None,
658
+ host: str | None = None,
659
+ share: bool | None = None,
660
+ server_port: int | None = None,
661
+ ):
662
+ """
663
+ Launches the Trackio dashboard.
664
+
665
+ Args:
666
+ project (`str`, *optional*):
667
+ The name of the project whose runs to show. If not provided, all projects
668
+ will be shown and the user can select one.
669
+ theme (`Any`, *optional*):
670
+ Ignored. Kept for backward compatibility; Trackio no longer uses Gradio themes.
671
+ mcp_server (`bool`, *optional*):
672
+ If `True`, the dashboard exposes an MCP server at `/mcp` when the optional
673
+ `trackio[mcp]` dependency is installed. If `None` (default), the
674
+ `GRADIO_MCP_SERVER` environment variable is used (e.g. on Spaces).
675
+ footer (`bool`, *optional*, defaults to `True`):
676
+ Whether to include `footer=false` in the write-token URL when `False`.
677
+ This can also be controlled via the `footer` query parameter in the URL.
678
+ color_palette (`list[str]`, *optional*):
679
+ A list of hex color codes to use for plot lines. If not provided, the
680
+ `TRACKIO_COLOR_PALETTE` environment variable will be used (comma-separated
681
+ hex codes), or if that is not set, the default color palette will be used.
682
+ Example: `['#FF0000', '#00FF00', '#0000FF']`
683
+ open_browser (`bool`, *optional*, defaults to `True`):
684
+ If `True` and not in a notebook, a new browser tab will be opened with the
685
+ dashboard. If `False`, the browser will not be opened.
686
+ block_thread (`bool`, *optional*):
687
+ If `True`, the main thread will be blocked until the dashboard is closed.
688
+ If `None` (default behavior), then the main thread will not be blocked if the
689
+ dashboard is launched in a notebook, otherwise the main thread will be blocked.
690
+ host (`str`, *optional*):
691
+ The host to bind the server to. If not provided, defaults to `'127.0.0.1'`
692
+ (localhost only). Set to `'0.0.0.0'` to allow remote access.
693
+ share (`bool`, *optional*):
694
+ If `True`, creates a temporary public URL (Gradio-compatible tunnel). On Colab
695
+ or hosted notebooks, defaults to `True` unless overridden.
696
+ server_port (`int`, *optional*):
697
+ Port to bind. If not set, scans from `GRADIO_SERVER_PORT` (default 7860).
698
+
699
+ Returns:
700
+ `app`: The dashboard handle (`.close()` stops the server).
701
+ `url`: The local URL of the dashboard.
702
+ `share_url`: The public share URL, if any.
703
+ `full_url`: The full URL including the write token (share URL when sharing, else local).
704
+ """
705
+ if theme is not None and theme != "default":
706
+ warnings.warn(
707
+ "The theme argument is ignored; Trackio no longer depends on Gradio themes.",
708
+ UserWarning,
709
+ stacklevel=2,
710
+ )
711
+
712
+ if color_palette is not None:
713
+ os.environ["TRACKIO_COLOR_PALETTE"] = ",".join(color_palette)
714
+
715
+ _mcp_server = (
716
+ mcp_server
717
+ if mcp_server is not None
718
+ else os.environ.get("GRADIO_MCP_SERVER", "False") == "True"
719
+ )
720
+
721
+ starlette_app, wt = build_starlette_app_only(mcp_server=_mcp_server)
722
+ local_url, share_url, _local_api_url, uv_server = launch_trackio_dashboard(
723
+ starlette_app,
724
+ server_name=host,
725
+ server_port=server_port,
726
+ share=share,
727
+ mcp_server=_mcp_server,
728
+ quiet=True,
729
+ )
730
+ server = TrackioDashboardApp(starlette_app, uv_server, wt)
731
+
732
+ base_root = (share_url or local_url).rstrip("/")
733
+ base_url = base_root + "/"
734
+ dashboard_url = base_url
735
+ if project:
736
+ dashboard_url += f"?project={project}"
737
+ full_url = utils.get_full_url(
738
+ base_root,
739
+ project=project,
740
+ write_token=wt,
741
+ footer=footer,
742
+ )
743
+
744
+ if not utils.is_in_notebook():
745
+ print(f"* Trackio UI launched at: {dashboard_url}")
746
+ if open_browser:
747
+ webbrowser.open(dashboard_url)
748
+ block_thread = block_thread if block_thread is not None else True
749
+ else:
750
+ utils.embed_url_in_notebook(dashboard_url)
751
+ block_thread = block_thread if block_thread is not None else False
752
+
753
+ if block_thread:
754
+ utils.block_main_thread_until_keyboard_interrupt()
755
+ return _TupleNoPrint((server, local_url, share_url, full_url))
trackio/_vendor/__init__.py ADDED
File without changes
trackio/_vendor/gradio_exceptions.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ class ChecksumMismatchError(Exception):
2
+ pass
3
+
4
+
5
+ class ShareCertificateWriteError(Exception):
6
+ pass
trackio/_vendor/networking.py ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import time
5
+ import warnings
6
+ from pathlib import Path
7
+ from urllib.parse import urlparse, urlunparse
8
+
9
+ import httpx
10
+
11
+ from trackio._vendor.gradio_exceptions import ShareCertificateWriteError
12
+ from trackio._vendor.tunneling import CERTIFICATE_PATH, Tunnel
13
+
14
+ GRADIO_API_SERVER = "https://api.gradio.app/v3/tunnel-request"
15
+ GRADIO_SHARE_SERVER_ADDRESS = os.getenv("GRADIO_SHARE_SERVER_ADDRESS")
16
+
17
+
18
+ def setup_tunnel(
19
+ local_host: str,
20
+ local_port: int,
21
+ share_token: str,
22
+ share_server_address: str | None,
23
+ share_server_tls_certificate: str | None,
24
+ ) -> str:
25
+ share_server_address = (
26
+ GRADIO_SHARE_SERVER_ADDRESS
27
+ if share_server_address is None
28
+ else share_server_address
29
+ )
30
+ if share_server_address is None:
31
+ try:
32
+ response = httpx.get(GRADIO_API_SERVER, timeout=30)
33
+ payload = response.json()[0]
34
+ remote_host, remote_port = payload["host"], int(payload["port"])
35
+ certificate = payload["root_ca"]
36
+ except Exception as e:
37
+ raise RuntimeError(
38
+ "Could not get share link from Gradio API Server."
39
+ ) from e
40
+ try:
41
+ Path(CERTIFICATE_PATH).parent.mkdir(parents=True, exist_ok=True)
42
+ with open(CERTIFICATE_PATH, "w") as f:
43
+ f.write(certificate)
44
+ except Exception as e:
45
+ raise ShareCertificateWriteError(
46
+ f"{e}. This can happen if the current working directory is read-only."
47
+ ) from e
48
+ share_server_tls_certificate = CERTIFICATE_PATH
49
+
50
+ else:
51
+ remote_host, remote_port = share_server_address.split(":")
52
+ remote_port = int(remote_port)
53
+ tunnel = Tunnel(
54
+ remote_host,
55
+ remote_port,
56
+ local_host,
57
+ local_port,
58
+ share_token,
59
+ share_server_tls_certificate,
60
+ )
61
+ address = tunnel.start_tunnel()
62
+ return address
63
+
64
+
65
+ def url_ok(url: str) -> bool:
66
+ try:
67
+ for _ in range(5):
68
+ with warnings.catch_warnings():
69
+ warnings.filterwarnings("ignore")
70
+ r = httpx.head(url, timeout=3, verify=False)
71
+ if r.status_code in (200, 401, 302, 303, 307):
72
+ return True
73
+ time.sleep(0.500)
74
+ except (ConnectionError, httpx.ConnectError, httpx.TimeoutException):
75
+ return False
76
+ return False
77
+
78
+
79
+ def normalize_share_url(share_url: str, share_server_protocol: str) -> str:
80
+ parsed_url = urlparse(share_url)
81
+ return urlunparse((share_server_protocol,) + parsed_url[1:])
trackio/_vendor/tunneling.py ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import atexit
2
+ import hashlib
3
+ import os
4
+ import platform
5
+ import re
6
+ import stat
7
+ import subprocess
8
+ import sys
9
+ import time
10
+ from pathlib import Path
11
+
12
+ import httpx
13
+ from huggingface_hub.constants import HF_HOME
14
+
15
+ from trackio._vendor.gradio_exceptions import ChecksumMismatchError
16
+
17
+ VERSION = "0.3"
18
+ CURRENT_TUNNELS: list["Tunnel"] = []
19
+
20
+ machine = platform.machine()
21
+ if machine == "x86_64":
22
+ machine = "amd64"
23
+ elif machine == "aarch64":
24
+ machine = "arm64"
25
+
26
+ BINARY_REMOTE_NAME = f"frpc_{platform.system().lower()}_{machine.lower()}"
27
+ EXTENSION = ".exe" if os.name == "nt" else ""
28
+ BINARY_URL = f"https://cdn-media.huggingface.co/frpc-gradio-{VERSION}/{BINARY_REMOTE_NAME}{EXTENSION}"
29
+
30
+ CHECKSUMS = {
31
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_windows_amd64.exe": "14bc0ea470be5d67d79a07412bd21de8a0a179c6ac1116d7764f68e942dc9ceb",
32
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_amd64": "c791d1f047b41ff5885772fc4bf20b797c6059bbd82abb9e31de15e55d6a57c4",
33
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_arm64": "823ced25104de6dc3c9f4798dbb43f20e681207279e6ab89c40e2176ccbf70cd",
34
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_darwin_amd64": "930f8face3365810ce16689da81b7d1941fda4466225a7bbcbced9a2916a6e15",
35
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_darwin_arm64": "dfac50c690aca459ed5158fad8bfbe99f9282baf4166cf7c410a6673fbc1f327",
36
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_arm": "4b563beb2e36c448cc688174e20b53af38dc1ff2b5e362d4ddd1401f2affbfb7",
37
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_freebsd_386": "cb0a56c764ecf96dd54ed601d240c564f060ee4e58202d65ffca17c1a51ce19c",
38
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_freebsd_amd64": "516d9e6903513869a011ddcd1ec206167ad1eb5dd6640d21057acc258edecbbb",
39
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_386": "4c2f2a48cd71571498c0ac8a4d42a055f22cb7f14b4b5a2b0d584220fd60a283",
40
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mips": "b309ecd594d4f0f7f33e556a80d4b67aef9319c00a8334648a618e56b23cb9e0",
41
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mips64": "0372ef5505baa6f3b64c6295a86541b24b7b0dbe4ef28b344992e21f47624b7b",
42
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_riscv64": "1658eed7e8c14ea76e1d95749d58441ce24147c3d559381832c725c29cfc3df3",
43
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mipsle": "a2aaba16961d3372b79bd7a28976fcd0f0bbaebc2b50d5a7a71af2240747960f",
44
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_windows_386.exe": "721b90550195a83e15f2176d8f85a48d5a25822757cb872e9723d4bccc4e5bb6",
45
+ "https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mips64le": "796481edd609f31962b45cc0ab4c9798d040205ae3bf354ed1b72fb432d796b8",
46
+ }
47
+
48
+ CHUNK_SIZE = 128
49
+
50
+ BINARY_FILENAME = f"{BINARY_REMOTE_NAME}_v{VERSION}"
51
+ BINARY_FOLDER = Path(HF_HOME) / "trackio" / "frpc"
52
+ BINARY_PATH = str(BINARY_FOLDER / BINARY_FILENAME)
53
+
54
+ TUNNEL_TIMEOUT_SECONDS = 30
55
+ TUNNEL_ERROR_MESSAGE = (
56
+ "Could not create share URL. "
57
+ "Please check the appended log from frpc for more information:"
58
+ )
59
+
60
+ CERTIFICATE_PATH = ".trackio/certificate.pem"
61
+
62
+
63
+ class Tunnel:
64
+ def __init__(
65
+ self,
66
+ remote_host: str,
67
+ remote_port: int,
68
+ local_host: str,
69
+ local_port: int,
70
+ share_token: str,
71
+ share_server_tls_certificate: str | None,
72
+ ):
73
+ self.proc = None
74
+ self.url = None
75
+ self.remote_host = remote_host
76
+ self.remote_port = remote_port
77
+ self.local_host = local_host
78
+ self.local_port = local_port
79
+ self.share_token = share_token
80
+ self.share_server_tls_certificate = share_server_tls_certificate
81
+
82
+ @staticmethod
83
+ def download_binary():
84
+ if not Path(BINARY_PATH).exists():
85
+ Path(BINARY_FOLDER).mkdir(parents=True, exist_ok=True)
86
+ resp = httpx.get(BINARY_URL, timeout=30)
87
+
88
+ if resp.status_code == 403:
89
+ raise OSError(
90
+ f"Cannot set up a share link as this platform is incompatible. Please "
91
+ f"create a GitHub issue with information about your platform: {platform.uname()}"
92
+ )
93
+
94
+ resp.raise_for_status()
95
+
96
+ with open(BINARY_PATH, "wb") as file:
97
+ file.write(resp.content)
98
+ st = os.stat(BINARY_PATH)
99
+ os.chmod(BINARY_PATH, st.st_mode | stat.S_IEXEC)
100
+
101
+ if BINARY_URL in CHECKSUMS:
102
+ sha = hashlib.sha256()
103
+ with open(BINARY_PATH, "rb") as f:
104
+ for chunk in iter(lambda: f.read(CHUNK_SIZE * sha.block_size), b""):
105
+ sha.update(chunk)
106
+ calculated_hash = sha.hexdigest()
107
+
108
+ if calculated_hash != CHECKSUMS[BINARY_URL]:
109
+ raise ChecksumMismatchError()
110
+
111
+ def start_tunnel(self) -> str:
112
+ self.download_binary()
113
+ self.url = self._start_tunnel(BINARY_PATH)
114
+ return self.url
115
+
116
+ def kill(self):
117
+ if self.proc is not None:
118
+ print(f"Killing tunnel {self.local_host}:{self.local_port} <> {self.url}")
119
+ self.proc.terminate()
120
+ self.proc = None
121
+
122
+ def _start_tunnel(self, binary: str) -> str:
123
+ CURRENT_TUNNELS.append(self)
124
+ command = [
125
+ binary,
126
+ "http",
127
+ "-n",
128
+ self.share_token,
129
+ "-l",
130
+ str(self.local_port),
131
+ "-i",
132
+ self.local_host,
133
+ "--uc",
134
+ "--sd",
135
+ "random",
136
+ "--ue",
137
+ "--server_addr",
138
+ f"{self.remote_host}:{self.remote_port}",
139
+ "--disable_log_color",
140
+ ]
141
+ if self.share_server_tls_certificate is not None:
142
+ command.extend(
143
+ [
144
+ "--tls_enable",
145
+ "--tls_trusted_ca_file",
146
+ self.share_server_tls_certificate,
147
+ ]
148
+ )
149
+ self.proc = subprocess.Popen(
150
+ command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
151
+ )
152
+ atexit.register(self.kill)
153
+ return self._read_url_from_tunnel_stream()
154
+
155
+ def _read_url_from_tunnel_stream(self) -> str:
156
+ start_timestamp = time.time()
157
+
158
+ log = []
159
+ url = ""
160
+
161
+ def _raise_tunnel_error():
162
+ log_text = "\n".join(log)
163
+ print(log_text, file=sys.stderr)
164
+ raise ValueError(f"{TUNNEL_ERROR_MESSAGE}\n{log_text}")
165
+
166
+ while url == "":
167
+ if time.time() - start_timestamp >= TUNNEL_TIMEOUT_SECONDS:
168
+ _raise_tunnel_error()
169
+
170
+ assert self.proc is not None # noqa: S101
171
+ if self.proc.stdout is None:
172
+ continue
173
+
174
+ line = self.proc.stdout.readline()
175
+ line = line.decode("utf-8")
176
+
177
+ if line == "":
178
+ continue
179
+
180
+ log.append(line.strip())
181
+
182
+ if "start proxy success" in line:
183
+ result = re.search("start proxy success: (.+)\n", line)
184
+ if result is None:
185
+ _raise_tunnel_error()
186
+ else:
187
+ url = result.group(1)
188
+ elif "login to server failed" in line:
189
+ _raise_tunnel_error()
190
+
191
+ return url
trackio/alerts.py ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import logging
3
+ import ssl
4
+ import urllib.request
5
+ from enum import Enum
6
+
7
+ try:
8
+ import certifi
9
+
10
+ _SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
11
+ except ImportError:
12
+ _SSL_CONTEXT = None
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ class AlertLevel(str, Enum):
18
+ INFO = "info"
19
+ WARN = "warn"
20
+ ERROR = "error"
21
+
22
+
23
+ ALERT_LEVEL_ORDER = {
24
+ AlertLevel.INFO: 0,
25
+ AlertLevel.WARN: 1,
26
+ AlertLevel.ERROR: 2,
27
+ }
28
+
29
+ ALERT_COLORS = {
30
+ AlertLevel.INFO: "\033[94m",
31
+ AlertLevel.WARN: "\033[93m",
32
+ AlertLevel.ERROR: "\033[91m",
33
+ }
34
+ RESET_COLOR = "\033[0m"
35
+
36
+ LEVEL_EMOJI = {
37
+ AlertLevel.INFO: "ℹ️",
38
+ AlertLevel.WARN: "⚠️",
39
+ AlertLevel.ERROR: "🚨",
40
+ }
41
+
42
+
43
+ def format_alert_terminal(
44
+ level: AlertLevel, title: str, text: str | None, step: int | None
45
+ ) -> str:
46
+ color = ALERT_COLORS.get(level, "")
47
+ step_str = f" (step {step})" if step is not None else ""
48
+ if text:
49
+ return f"{color}[TRACKIO {level.value.upper()}]{RESET_COLOR} {title}: {text}{step_str}"
50
+ return f"{color}[TRACKIO {level.value.upper()}]{RESET_COLOR} {title}{step_str}"
51
+
52
+
53
+ def _is_slack_url(url: str) -> bool:
54
+ return "hooks.slack.com" in url
55
+
56
+
57
+ def _is_discord_url(url: str) -> bool:
58
+ return "discord.com/api/webhooks" in url or "discordapp.com/api/webhooks" in url
59
+
60
+
61
+ def _build_slack_payload(
62
+ level: AlertLevel,
63
+ title: str,
64
+ text: str | None,
65
+ project: str,
66
+ run: str,
67
+ step: int | None,
68
+ ) -> dict:
69
+ emoji = LEVEL_EMOJI.get(level, "")
70
+ step_str = f" • Step {step}" if step is not None else ""
71
+ header = f"{emoji} *[{level.value.upper()}] {title}*"
72
+ context = f"Project: {project} • Run: {run}{step_str}"
73
+ blocks = [
74
+ {"type": "section", "text": {"type": "mrkdwn", "text": header}},
75
+ ]
76
+ if text:
77
+ blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": text}})
78
+ blocks.append(
79
+ {"type": "context", "elements": [{"type": "mrkdwn", "text": context}]}
80
+ )
81
+ return {"blocks": blocks}
82
+
83
+
84
+ def _build_discord_payload(
85
+ level: AlertLevel,
86
+ title: str,
87
+ text: str | None,
88
+ project: str,
89
+ run: str,
90
+ step: int | None,
91
+ ) -> dict:
92
+ color_map = {
93
+ AlertLevel.INFO: 3447003,
94
+ AlertLevel.WARN: 16776960,
95
+ AlertLevel.ERROR: 15158332,
96
+ }
97
+ emoji = LEVEL_EMOJI.get(level, "")
98
+ step_str = f" • Step {step}" if step is not None else ""
99
+ embed = {
100
+ "title": f"{emoji} [{level.value.upper()}] {title}",
101
+ "color": color_map.get(level, 0),
102
+ "footer": {"text": f"Project: {project} • Run: {run}{step_str}"},
103
+ }
104
+ if text:
105
+ embed["description"] = text
106
+ return {"embeds": [embed]}
107
+
108
+
109
+ def _build_generic_payload(
110
+ level: AlertLevel,
111
+ title: str,
112
+ text: str | None,
113
+ project: str,
114
+ run: str,
115
+ step: int | None,
116
+ timestamp: str | None,
117
+ ) -> dict:
118
+ return {
119
+ "level": level.value,
120
+ "title": title,
121
+ "text": text,
122
+ "project": project,
123
+ "run": run,
124
+ "step": step,
125
+ "timestamp": timestamp,
126
+ }
127
+
128
+
129
+ def parse_alert_level(level: AlertLevel | str) -> AlertLevel:
130
+ if isinstance(level, AlertLevel):
131
+ return level
132
+ normalized = level.lower().strip()
133
+ try:
134
+ return AlertLevel(normalized)
135
+ except ValueError as e:
136
+ allowed = ", ".join(lvl.value for lvl in AlertLevel)
137
+ raise ValueError(
138
+ f"Invalid alert level '{level}'. Expected one of: {allowed}."
139
+ ) from e
140
+
141
+
142
+ def resolve_webhook_min_level(
143
+ webhook_min_level: AlertLevel | str | None,
144
+ ) -> AlertLevel | None:
145
+ if webhook_min_level is None:
146
+ return None
147
+ return parse_alert_level(webhook_min_level)
148
+
149
+
150
+ def should_send_webhook(
151
+ level: AlertLevel, webhook_min_level: AlertLevel | None
152
+ ) -> bool:
153
+ if webhook_min_level is None:
154
+ return True
155
+ return ALERT_LEVEL_ORDER[level] >= ALERT_LEVEL_ORDER[webhook_min_level]
156
+
157
+
158
+ def send_webhook(
159
+ url: str,
160
+ level: AlertLevel,
161
+ title: str,
162
+ text: str | None,
163
+ project: str,
164
+ run: str,
165
+ step: int | None,
166
+ timestamp: str | None = None,
167
+ ) -> None:
168
+ if _is_slack_url(url):
169
+ payload = _build_slack_payload(level, title, text, project, run, step)
170
+ elif _is_discord_url(url):
171
+ payload = _build_discord_payload(level, title, text, project, run, step)
172
+ else:
173
+ payload = _build_generic_payload(
174
+ level, title, text, project, run, step, timestamp
175
+ )
176
+
177
+ data = json.dumps(payload).encode("utf-8")
178
+ req = urllib.request.Request(
179
+ url, data=data, headers={"Content-Type": "application/json"}
180
+ )
181
+ try:
182
+ urllib.request.urlopen(req, timeout=10, context=_SSL_CONTEXT)
183
+ except Exception as e:
184
+ logger.warning(f"Failed to send webhook to {url}: {e}")
trackio/api.py ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Iterator
2
+
3
+ from trackio.sqlite_storage import SQLiteStorage
4
+
5
+
6
+ class Run:
7
+ def __init__(self, project: str, name: str):
8
+ self.project = project
9
+ self.name = name
10
+ self._config = None
11
+
12
+ @property
13
+ def id(self) -> str:
14
+ return self.name
15
+
16
+ @property
17
+ def config(self) -> dict | None:
18
+ if self._config is None:
19
+ self._config = SQLiteStorage.get_run_config(self.project, self.name)
20
+ return self._config
21
+
22
+ def alerts(self, level: str | None = None, since: str | None = None) -> list[dict]:
23
+ return SQLiteStorage.get_alerts(
24
+ self.project, run_name=self.name, level=level, since=since
25
+ )
26
+
27
+ def delete(self) -> bool:
28
+ return SQLiteStorage.delete_run(self.project, self.name)
29
+
30
+ def move(self, new_project: str) -> bool:
31
+ success = SQLiteStorage.move_run(self.project, self.name, new_project)
32
+ if success:
33
+ self.project = new_project
34
+ return success
35
+
36
+ def rename(self, new_name: str) -> "Run":
37
+ SQLiteStorage.rename_run(self.project, self.name, new_name)
38
+ self.name = new_name
39
+ return self
40
+
41
+ def __repr__(self) -> str:
42
+ return f"<Run {self.name} in project {self.project}>"
43
+
44
+
45
+ class Runs:
46
+ def __init__(self, project: str):
47
+ self.project = project
48
+ self._runs = None
49
+
50
+ def _load_runs(self):
51
+ if self._runs is None:
52
+ run_names = SQLiteStorage.get_runs(self.project)
53
+ self._runs = [Run(self.project, name) for name in run_names]
54
+
55
+ def __iter__(self) -> Iterator[Run]:
56
+ self._load_runs()
57
+ return iter(self._runs)
58
+
59
+ def __getitem__(self, index: int) -> Run:
60
+ self._load_runs()
61
+ return self._runs[index]
62
+
63
+ def __len__(self) -> int:
64
+ self._load_runs()
65
+ return len(self._runs)
66
+
67
+ def __repr__(self) -> str:
68
+ self._load_runs()
69
+ return f"<Runs project={self.project} count={len(self._runs)}>"
70
+
71
+
72
+ class Api:
73
+ def runs(self, project: str) -> Runs:
74
+ if not SQLiteStorage.get_project_db_path(project).exists():
75
+ raise ValueError(f"Project '{project}' does not exist")
76
+ return Runs(project)
77
+
78
+ def alerts(
79
+ self,
80
+ project: str,
81
+ run: str | None = None,
82
+ level: str | None = None,
83
+ since: str | None = None,
84
+ ) -> list[dict]:
85
+ if not SQLiteStorage.get_project_db_path(project).exists():
86
+ raise ValueError(f"Project '{project}' does not exist")
87
+ return SQLiteStorage.get_alerts(project, run_name=run, level=level, since=since)
trackio/apple_gpu.py ADDED
@@ -0,0 +1,253 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import platform
2
+ import subprocess
3
+ import sys
4
+ import threading
5
+ import warnings
6
+ from typing import TYPE_CHECKING, Any
7
+
8
+ if TYPE_CHECKING:
9
+ from trackio.run import Run
10
+
11
+ psutil: Any = None
12
+ PSUTIL_AVAILABLE = False
13
+ _monitor_lock = threading.Lock()
14
+
15
+
16
+ def _ensure_psutil():
17
+ global PSUTIL_AVAILABLE, psutil
18
+ if PSUTIL_AVAILABLE:
19
+ return psutil
20
+ try:
21
+ import psutil as _psutil
22
+
23
+ psutil = _psutil
24
+ PSUTIL_AVAILABLE = True
25
+ return psutil
26
+ except ImportError:
27
+ raise ImportError(
28
+ "psutil is required for Apple Silicon monitoring. "
29
+ "Install it with: pip install psutil"
30
+ )
31
+
32
+
33
+ def is_apple_silicon() -> bool:
34
+ """Check if running on Apple Silicon (M1/M2/M3/M4)."""
35
+ if platform.system() != "Darwin":
36
+ return False
37
+
38
+ try:
39
+ result = subprocess.run(
40
+ ["sysctl", "-n", "machdep.cpu.brand_string"],
41
+ capture_output=True,
42
+ text=True,
43
+ timeout=1,
44
+ )
45
+ cpu_brand = result.stdout.strip()
46
+ return "Apple" in cpu_brand
47
+ except Exception:
48
+ return False
49
+
50
+
51
+ def get_gpu_info() -> dict[str, Any]:
52
+ """Get Apple GPU information using ioreg."""
53
+ try:
54
+ result = subprocess.run(
55
+ ["ioreg", "-r", "-d", "1", "-w", "0", "-c", "IOAccelerator"],
56
+ capture_output=True,
57
+ text=True,
58
+ timeout=2,
59
+ )
60
+
61
+ if result.returncode == 0 and result.stdout:
62
+ lines = result.stdout.strip().split("\n")
63
+ for line in lines:
64
+ if "IOAccelerator" in line and "class" in line:
65
+ return {"detected": True, "type": "Apple GPU"}
66
+ else:
67
+ print("Error collecting Apple GPU info. ioreg stdout was:", file=sys.stderr)
68
+ print(result.stdout, file=sys.stderr)
69
+ print("ioreg stderr was:", file=sys.stderr)
70
+ print(result.stderr, file=sys.stderr)
71
+
72
+ result = subprocess.run(
73
+ ["system_profiler", "SPDisplaysDataType"],
74
+ capture_output=True,
75
+ text=True,
76
+ timeout=3,
77
+ )
78
+
79
+ if result.returncode == 0 and "Apple" in result.stdout:
80
+ for line in result.stdout.split("\n"):
81
+ if "Chipset Model:" in line:
82
+ model = line.split(":")[-1].strip()
83
+ return {"detected": True, "type": model}
84
+
85
+ except Exception:
86
+ pass
87
+
88
+ return {"detected": False}
89
+
90
+
91
+ def apple_gpu_available() -> bool:
92
+ """
93
+ Check if Apple GPU monitoring is available.
94
+
95
+ Returns True if running on Apple Silicon (M-series chips) and psutil is installed.
96
+ """
97
+ try:
98
+ _ensure_psutil()
99
+ return is_apple_silicon()
100
+ except ImportError:
101
+ return False
102
+ except Exception:
103
+ return False
104
+
105
+
106
+ def collect_apple_metrics() -> dict:
107
+ """
108
+ Collect system metrics for Apple Silicon.
109
+
110
+ Returns:
111
+ Dictionary of system metrics including CPU, memory, and GPU info.
112
+ """
113
+ if not PSUTIL_AVAILABLE:
114
+ try:
115
+ _ensure_psutil()
116
+ except ImportError:
117
+ return {}
118
+
119
+ metrics = {}
120
+
121
+ try:
122
+ cpu_percent = psutil.cpu_percent(interval=0.1, percpu=False)
123
+ metrics["cpu/utilization"] = cpu_percent
124
+ except Exception:
125
+ pass
126
+
127
+ try:
128
+ cpu_percents = psutil.cpu_percent(interval=0.1, percpu=True)
129
+ for i, percent in enumerate(cpu_percents):
130
+ metrics[f"cpu/{i}/utilization"] = percent
131
+ except Exception:
132
+ pass
133
+
134
+ try:
135
+ cpu_freq = psutil.cpu_freq()
136
+ if cpu_freq:
137
+ metrics["cpu/frequency"] = cpu_freq.current
138
+ if cpu_freq.max > 0:
139
+ metrics["cpu/frequency_max"] = cpu_freq.max
140
+ except Exception:
141
+ pass
142
+
143
+ try:
144
+ mem = psutil.virtual_memory()
145
+ metrics["memory/used"] = mem.used / (1024**3)
146
+ metrics["memory/total"] = mem.total / (1024**3)
147
+ metrics["memory/available"] = mem.available / (1024**3)
148
+ metrics["memory/percent"] = mem.percent
149
+ except Exception:
150
+ pass
151
+
152
+ try:
153
+ swap = psutil.swap_memory()
154
+ metrics["swap/used"] = swap.used / (1024**3)
155
+ metrics["swap/total"] = swap.total / (1024**3)
156
+ metrics["swap/percent"] = swap.percent
157
+ except Exception:
158
+ pass
159
+
160
+ try:
161
+ sensors_temps = psutil.sensors_temperatures()
162
+ if sensors_temps:
163
+ for name, entries in sensors_temps.items():
164
+ for i, entry in enumerate(entries):
165
+ label = entry.label or f"{name}_{i}"
166
+ metrics[f"temp/{label}"] = entry.current
167
+ except Exception:
168
+ pass
169
+
170
+ gpu_info = get_gpu_info()
171
+ if gpu_info.get("detected"):
172
+ metrics["gpu/detected"] = 1
173
+ if "type" in gpu_info:
174
+ pass
175
+
176
+ return metrics
177
+
178
+
179
+ class AppleGpuMonitor:
180
+ def __init__(self, run: "Run", interval: float = 10.0):
181
+ self._run = run
182
+ self._interval = interval
183
+ self._stop_flag = threading.Event()
184
+ self._thread: "threading.Thread | None" = None
185
+
186
+ def start(self):
187
+ if not is_apple_silicon():
188
+ warnings.warn(
189
+ "auto_log_gpu=True but not running on Apple Silicon. "
190
+ "Apple GPU logging disabled."
191
+ )
192
+ return
193
+
194
+ if not PSUTIL_AVAILABLE:
195
+ try:
196
+ _ensure_psutil()
197
+ except ImportError:
198
+ warnings.warn(
199
+ "auto_log_gpu=True but psutil not installed. "
200
+ "Install with: pip install psutil"
201
+ )
202
+ return
203
+
204
+ self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
205
+ self._thread.start()
206
+
207
+ def stop(self):
208
+ self._stop_flag.set()
209
+ if self._thread is not None:
210
+ self._thread.join(timeout=2.0)
211
+
212
+ def _monitor_loop(self):
213
+ while not self._stop_flag.is_set():
214
+ try:
215
+ metrics = collect_apple_metrics()
216
+ if metrics:
217
+ self._run.log_system(metrics)
218
+ except Exception:
219
+ pass
220
+
221
+ self._stop_flag.wait(timeout=self._interval)
222
+
223
+
224
+ def log_apple_gpu(run: "Run | None" = None) -> dict:
225
+ """
226
+ Log Apple Silicon system metrics to the current or specified run.
227
+
228
+ Args:
229
+ run: Optional Run instance. If None, uses current run from context.
230
+
231
+ Returns:
232
+ dict: The system metrics that were logged.
233
+
234
+ Example:
235
+ ```python
236
+ import trackio
237
+
238
+ run = trackio.init(project="my-project")
239
+ trackio.log({"loss": 0.5})
240
+ trackio.log_apple_gpu()
241
+ ```
242
+ """
243
+ from trackio import context_vars
244
+
245
+ if run is None:
246
+ run = context_vars.current_run.get()
247
+ if run is None:
248
+ raise RuntimeError("Call trackio.init() before trackio.log_apple_gpu().")
249
+
250
+ metrics = collect_apple_metrics()
251
+ if metrics:
252
+ run.log_system(metrics)
253
+ return metrics
trackio/asgi_app.py ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import inspect
4
+ import json
5
+ import math
6
+ import tempfile
7
+ import threading
8
+ from pathlib import Path
9
+ from typing import Any
10
+ from urllib.parse import unquote
11
+
12
+ from starlette.applications import Starlette
13
+ from starlette.requests import Request
14
+ from starlette.responses import FileResponse, JSONResponse, Response
15
+ from starlette.routing import Route
16
+
17
+ from trackio.exceptions import TrackioAPIError
18
+ from trackio.remote_client import HTTP_API_VERSION
19
+
20
+ _PACKAGE_JSON_PATH = Path(__file__).parent / "package.json"
21
+ _TRACKIO_PACKAGE_VERSION = json.loads(_PACKAGE_JSON_PATH.read_text())["version"]
22
+
23
+
24
+ def _normalize_allowed_file_roots(
25
+ allowed_file_roots: list[str | Path] | None,
26
+ ) -> tuple[Path, ...]:
27
+ roots = []
28
+ for root in allowed_file_roots or []:
29
+ roots.append(Path(root).resolve())
30
+ return tuple(roots)
31
+
32
+
33
+ def _is_allowed_file_path(path: Path, allowed_roots: tuple[Path, ...]) -> bool:
34
+ resolved_path = path.resolve(strict=False)
35
+ for root in allowed_roots:
36
+ try:
37
+ resolved_path.relative_to(root)
38
+ return True
39
+ except ValueError:
40
+ continue
41
+ return False
42
+
43
+
44
+ def _json_safe(data: Any) -> Any:
45
+ if data is None or isinstance(data, (str, bool, int)):
46
+ return data
47
+ if isinstance(data, float):
48
+ return data if math.isfinite(data) else None
49
+ if isinstance(data, dict):
50
+ return {k: _json_safe(v) for k, v in data.items()}
51
+ if isinstance(data, (list, tuple)):
52
+ return [_json_safe(v) for v in data]
53
+ if hasattr(data, "item"):
54
+ try:
55
+ return _json_safe(data.item())
56
+ except Exception:
57
+ pass
58
+ return str(data)
59
+
60
+
61
+ def register_uploaded_temp_file(request: Request, file_path: str | Path) -> None:
62
+ resolved_path = Path(file_path).resolve(strict=False)
63
+ with request.app.state.uploaded_temp_files_lock:
64
+ request.app.state.uploaded_temp_files.add(resolved_path)
65
+
66
+
67
+ def consume_uploaded_temp_file(request: Request, file_data: Any) -> Path:
68
+ file_path = file_data.get("path") if isinstance(file_data, dict) else None
69
+ if not isinstance(file_path, str) or not file_path:
70
+ raise TrackioAPIError("Expected uploaded file metadata with a valid path.")
71
+
72
+ resolved_path = Path(file_path).resolve(strict=False)
73
+ with request.app.state.uploaded_temp_files_lock:
74
+ if resolved_path not in request.app.state.uploaded_temp_files:
75
+ raise TrackioAPIError(
76
+ "Uploaded file was not created by this Trackio server."
77
+ )
78
+ request.app.state.uploaded_temp_files.remove(resolved_path)
79
+
80
+ if not resolved_path.is_file():
81
+ raise TrackioAPIError("Uploaded file is missing.")
82
+
83
+ return resolved_path
84
+
85
+
86
+ def _invoke_handler(
87
+ fn: Any,
88
+ request: Request,
89
+ args: list[Any] | None = None,
90
+ kwargs: dict[str, Any] | None = None,
91
+ ) -> Any:
92
+ sig = inspect.signature(fn)
93
+ params = list(sig.parameters.values())
94
+ positional_args: list[Any] = []
95
+ keyword_args: dict[str, Any] = {}
96
+ args = list(args or [])
97
+ kwargs = dict(kwargs or {})
98
+ data_index = 0
99
+
100
+ for param in params:
101
+ if param.name == "request":
102
+ keyword_args["request"] = request
103
+ elif param.kind == inspect.Parameter.VAR_POSITIONAL:
104
+ positional_args.extend(args[data_index:])
105
+ data_index = len(args)
106
+ elif param.kind == inspect.Parameter.VAR_KEYWORD:
107
+ keyword_args.update(kwargs)
108
+ kwargs.clear()
109
+ elif param.name in kwargs:
110
+ keyword_args[param.name] = kwargs.pop(param.name)
111
+ elif param.kind in (
112
+ inspect.Parameter.POSITIONAL_ONLY,
113
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
114
+ ) and data_index < len(args):
115
+ positional_args.append(args[data_index])
116
+ data_index += 1
117
+ elif param.default is inspect.Signature.empty and param.kind not in (
118
+ inspect.Parameter.VAR_POSITIONAL,
119
+ inspect.Parameter.VAR_KEYWORD,
120
+ ):
121
+ raise TrackioAPIError(f"Missing required parameter: {param.name}")
122
+
123
+ return fn(*positional_args, **keyword_args)
124
+
125
+
126
+ async def version_handler(request: Request) -> Response:
127
+ mcp_enabled = bool(getattr(request.app.state, "mcp_enabled", False))
128
+ return JSONResponse(
129
+ {
130
+ "version": _TRACKIO_PACKAGE_VERSION,
131
+ "api_version": HTTP_API_VERSION,
132
+ "api_transport": "http",
133
+ "mcp_enabled": mcp_enabled,
134
+ "mcp_path": "/mcp" if mcp_enabled else None,
135
+ }
136
+ )
137
+
138
+
139
+ async def api_handler(request: Request) -> Response:
140
+ api_registry = request.app.state.api_registry
141
+ api_name = request.path_params["api_name"]
142
+ fn = api_registry.get(api_name)
143
+ if fn is None:
144
+ return JSONResponse({"error": f"Unknown API: {api_name}"}, status_code=404)
145
+
146
+ try:
147
+ body = await request.json()
148
+ except Exception:
149
+ body = {}
150
+
151
+ args: list[Any] = []
152
+ kwargs: dict[str, Any] = {}
153
+ if isinstance(body, dict):
154
+ if "args" in body or "kwargs" in body:
155
+ args = body.get("args") or []
156
+ kwargs = body.get("kwargs") or {}
157
+ elif "data" in body and isinstance(body["data"], list):
158
+ args = body["data"]
159
+ else:
160
+ kwargs = body
161
+ elif isinstance(body, list):
162
+ args = body
163
+ elif body is not None:
164
+ args = [body]
165
+
166
+ if not isinstance(args, list):
167
+ args = [args]
168
+ if not isinstance(kwargs, dict):
169
+ kwargs = {}
170
+
171
+ try:
172
+ result = _invoke_handler(fn, request, args=args, kwargs=kwargs)
173
+ return JSONResponse({"data": _json_safe(result)})
174
+ except TrackioAPIError as e:
175
+ return JSONResponse({"error": str(e)}, status_code=400)
176
+ except Exception as e:
177
+ return JSONResponse({"error": str(e)}, status_code=500)
178
+
179
+
180
+ async def upload_handler(request: Request) -> Response:
181
+ form = await request.form()
182
+ uploads = form.getlist("files")
183
+ saved_paths = []
184
+ for upload in uploads:
185
+ suffix = Path(getattr(upload, "filename", "") or "").suffix
186
+ with tempfile.NamedTemporaryFile(
187
+ delete=False,
188
+ prefix="trackio-upload-",
189
+ suffix=suffix,
190
+ ) as tmp:
191
+ tmp.write(await upload.read())
192
+ register_uploaded_temp_file(request, tmp.name)
193
+ saved_paths.append(tmp.name)
194
+ return JSONResponse({"paths": saved_paths})
195
+
196
+
197
+ async def file_handler(request: Request) -> Response:
198
+ fs_path = request.query_params.get("path")
199
+ if fs_path is None:
200
+ return Response("Missing path", status_code=400)
201
+ fp = Path(unquote(fs_path)).resolve(strict=False)
202
+ allowed_roots = getattr(request.app.state, "allowed_file_roots", ())
203
+ if fp.is_file() and _is_allowed_file_path(fp, allowed_roots):
204
+ return FileResponse(str(fp))
205
+ return Response("Not found", status_code=404)
206
+
207
+
208
+ def create_trackio_starlette_app(
209
+ oauth_routes: list[Route],
210
+ api_registry: dict[str, Any],
211
+ extra_routes: list[Any] | None = None,
212
+ mcp_lifespan: Any = None,
213
+ mcp_enabled: bool = False,
214
+ allowed_file_roots: list[str | Path] | None = None,
215
+ ) -> Starlette:
216
+ routes: list[Any] = list(oauth_routes)
217
+ routes.extend(
218
+ [
219
+ Route("/version", endpoint=version_handler, methods=["GET"]),
220
+ Route("/api/upload", endpoint=upload_handler, methods=["POST"]),
221
+ Route("/api/{api_name:str}", endpoint=api_handler, methods=["POST"]),
222
+ Route("/file", endpoint=file_handler, methods=["GET"]),
223
+ ]
224
+ )
225
+ routes.extend(extra_routes or [])
226
+ app = Starlette(routes=routes, lifespan=mcp_lifespan)
227
+ app.state.api_registry = api_registry
228
+ app.state.mcp_enabled = mcp_enabled
229
+ app.state.allowed_file_roots = _normalize_allowed_file_roots(allowed_file_roots)
230
+ app.state.uploaded_temp_files = set()
231
+ app.state.uploaded_temp_files_lock = threading.Lock()
232
+ return app
trackio/assets/badge.png ADDED

Git LFS Details

  • SHA256: 206b7847247e83279f498510a2760338a03116bb5141a658d71ec14429f9ea9e
  • Pointer size: 131 Bytes
  • Size of remote file: 170 kB
trackio/assets/trackio_logo_dark.png ADDED
trackio/assets/trackio_logo_light.png ADDED
trackio/assets/trackio_logo_old.png ADDED

Git LFS Details

  • SHA256: 3922c4d1e465270ad4d8abb12023f3beed5d9f7f338528a4c0ac21dcf358a1c8
  • Pointer size: 131 Bytes
  • Size of remote file: 487 kB
trackio/assets/trackio_logo_type_dark.png ADDED
trackio/assets/trackio_logo_type_dark_transparent.png ADDED
trackio/assets/trackio_logo_type_light.png ADDED
trackio/assets/trackio_logo_type_light_transparent.png ADDED
trackio/bucket_storage.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import shutil
2
+ import tempfile
3
+ from pathlib import Path
4
+
5
+ import huggingface_hub
6
+ from huggingface_hub import copy_files, sync_bucket
7
+
8
+ from trackio.sqlite_storage import SQLiteStorage
9
+ from trackio.utils import MEDIA_DIR, TRACKIO_DIR
10
+
11
+
12
+ def create_bucket_if_not_exists(bucket_id: str, private: bool | None = None) -> None:
13
+ huggingface_hub.create_bucket(bucket_id, private=private, exist_ok=True)
14
+
15
+
16
+ def _list_bucket_file_paths(bucket_id: str, prefix: str | None = None) -> list[str]:
17
+ items = huggingface_hub.list_bucket_tree(bucket_id, prefix=prefix, recursive=True)
18
+ return [
19
+ item.path
20
+ for item in items
21
+ if getattr(item, "type", None) == "file" and getattr(item, "path", None)
22
+ ]
23
+
24
+
25
+ def download_bucket_to_trackio_dir(bucket_id: str) -> None:
26
+ TRACKIO_DIR.mkdir(parents=True, exist_ok=True)
27
+ sync_bucket(
28
+ source=f"hf://buckets/{bucket_id}",
29
+ dest=str(TRACKIO_DIR.parent),
30
+ quiet=True,
31
+ )
32
+
33
+
34
+ def upload_project_to_bucket(project: str, bucket_id: str) -> None:
35
+ db_path = SQLiteStorage.get_project_db_path(project)
36
+ if not db_path.exists():
37
+ raise FileNotFoundError(f"No database found for project '{project}'")
38
+
39
+ with SQLiteStorage._get_connection(
40
+ db_path, configure_pragmas=False, row_factory=None
41
+ ) as conn:
42
+ conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
43
+
44
+ files_to_add = [(str(db_path), f"trackio/{db_path.name}")]
45
+
46
+ media_dir = MEDIA_DIR / project
47
+ if media_dir.exists():
48
+ for media_file in media_dir.rglob("*"):
49
+ if media_file.is_file():
50
+ rel = media_file.relative_to(TRACKIO_DIR)
51
+ files_to_add.append((str(media_file), f"trackio/{rel}"))
52
+
53
+ huggingface_hub.batch_bucket_files(bucket_id, add=files_to_add)
54
+
55
+
56
+ def _download_db_from_bucket(
57
+ project: str, bucket_id: str, dest_path: Path | None = None
58
+ ) -> bool:
59
+ db_filename = SQLiteStorage.get_project_db_filename(project)
60
+ remote_path = f"trackio/{db_filename}"
61
+ local_path = dest_path or SQLiteStorage.get_project_db_path(project)
62
+ local_path.parent.mkdir(parents=True, exist_ok=True)
63
+ try:
64
+ huggingface_hub.download_bucket_files(
65
+ bucket_id,
66
+ files=[(remote_path, str(local_path))],
67
+ token=huggingface_hub.utils.get_token(),
68
+ )
69
+ return local_path.exists()
70
+ except Exception:
71
+ return False
72
+
73
+
74
+ def _local_db_has_data(project: str) -> bool:
75
+ db_path = SQLiteStorage.get_project_db_path(project)
76
+ if not db_path.exists() or db_path.stat().st_size == 0:
77
+ return False
78
+ try:
79
+ with SQLiteStorage._get_connection(
80
+ db_path, configure_pragmas=False, row_factory=None
81
+ ) as conn:
82
+ count = conn.execute("SELECT COUNT(*) FROM metrics").fetchone()[0]
83
+ return count > 0
84
+ except Exception:
85
+ return False
86
+
87
+
88
+ def _export_and_upload_static(
89
+ project: str,
90
+ dest_bucket_id: str,
91
+ db_path: Path,
92
+ media_dir: Path | None = None,
93
+ ) -> None:
94
+ with tempfile.TemporaryDirectory() as tmp_dir:
95
+ output_dir = Path(tmp_dir)
96
+ SQLiteStorage.export_for_static_space(
97
+ project, output_dir, db_path_override=db_path
98
+ )
99
+
100
+ if media_dir and media_dir.exists():
101
+ shutil.copytree(media_dir, output_dir / "media")
102
+
103
+ files_to_add = []
104
+ for f in output_dir.rglob("*"):
105
+ if f.is_file():
106
+ rel = f.relative_to(output_dir)
107
+ files_to_add.append((str(f), str(rel)))
108
+
109
+ huggingface_hub.batch_bucket_files(dest_bucket_id, add=files_to_add)
110
+
111
+
112
+ def _copy_project_media_between_buckets(
113
+ source_bucket_id: str, dest_bucket_id: str, project: str
114
+ ) -> None:
115
+ source_media_prefix = f"trackio/media/{project}/"
116
+ media_to_copy = _list_bucket_file_paths(
117
+ source_bucket_id, prefix=source_media_prefix
118
+ )
119
+ if not media_to_copy:
120
+ return
121
+
122
+ copy_files(
123
+ f"hf://buckets/{source_bucket_id}/{source_media_prefix}",
124
+ f"hf://buckets/{dest_bucket_id}/media/",
125
+ )
126
+
127
+
128
+ def upload_project_to_bucket_for_static(project: str, bucket_id: str) -> None:
129
+ if not _local_db_has_data(project):
130
+ _download_db_from_bucket(project, bucket_id)
131
+
132
+ db_path = SQLiteStorage.get_project_db_path(project)
133
+ _export_and_upload_static(project, bucket_id, db_path, MEDIA_DIR / project)
134
+
135
+
136
+ def export_from_bucket_for_static(
137
+ source_bucket_id: str,
138
+ dest_bucket_id: str,
139
+ project: str,
140
+ ) -> None:
141
+ with tempfile.TemporaryDirectory() as work_dir:
142
+ work_path = Path(work_dir)
143
+ db_path = work_path / SQLiteStorage.get_project_db_filename(project)
144
+
145
+ if not _download_db_from_bucket(project, source_bucket_id, dest_path=db_path):
146
+ raise FileNotFoundError(
147
+ f"Could not download database for project '{project}' "
148
+ f"from bucket '{source_bucket_id}'."
149
+ )
150
+
151
+ _export_and_upload_static(project, dest_bucket_id, db_path)
152
+ _copy_project_media_between_buckets(source_bucket_id, dest_bucket_id, project)
trackio/cli.py ADDED
@@ -0,0 +1,1272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import os
3
+
4
+ from trackio import freeze, show, sync
5
+ from trackio.cli_helpers import (
6
+ error_exit,
7
+ format_alerts,
8
+ format_json,
9
+ format_list,
10
+ format_metric_values,
11
+ format_project_summary,
12
+ format_run_summary,
13
+ format_snapshot,
14
+ format_system_metric_names,
15
+ format_system_metrics,
16
+ )
17
+ from trackio.markdown import Markdown
18
+ from trackio.server import get_project_summary, get_run_summary
19
+ from trackio.sqlite_storage import SQLiteStorage
20
+
21
+
22
+ def _get_space(args):
23
+ return getattr(args, "space", None)
24
+
25
+
26
+ def _get_remote(args):
27
+ from trackio.remote_client import RemoteClient
28
+
29
+ space = _get_space(args)
30
+ if not space:
31
+ return None
32
+ hf_token = getattr(args, "hf_token", None)
33
+ return RemoteClient(space, hf_token=hf_token)
34
+
35
+
36
+ def _handle_status():
37
+ print("Reading local Trackio projects...\n")
38
+ projects = SQLiteStorage.get_projects()
39
+ if not projects:
40
+ print("No Trackio projects found.")
41
+ return
42
+
43
+ local_projects = []
44
+ synced_projects = []
45
+ unsynced_projects = []
46
+
47
+ for project in projects:
48
+ space_id = SQLiteStorage.get_space_id(project)
49
+ if space_id is None:
50
+ local_projects.append(project)
51
+ elif SQLiteStorage.has_pending_data(project):
52
+ unsynced_projects.append(project)
53
+ else:
54
+ synced_projects.append(project)
55
+
56
+ print("Finished reading Trackio projects")
57
+ if local_projects:
58
+ print(f" * {len(local_projects)} local trackio project(s) [OK]")
59
+ if synced_projects:
60
+ print(f" * {len(synced_projects)} trackio project(s) synced to Spaces [OK]")
61
+ if unsynced_projects:
62
+ print(
63
+ f" * {len(unsynced_projects)} trackio project(s) with unsynced changes [WARNING]:"
64
+ )
65
+ for p in unsynced_projects:
66
+ print(f" - {p}")
67
+
68
+ if unsynced_projects:
69
+ print(
70
+ f"\nRun `trackio sync --project {unsynced_projects[0]}` to sync. "
71
+ "Or run `trackio sync --all` to sync all unsynced changes."
72
+ )
73
+
74
+
75
+ def _handle_sync(args):
76
+ from trackio.deploy import sync_incremental
77
+
78
+ if args.sync_all and args.project:
79
+ error_exit("Cannot use --all and --project together.")
80
+ if not args.sync_all and not args.project:
81
+ error_exit("Must provide either --project or --all.")
82
+
83
+ if args.sync_all:
84
+ projects = SQLiteStorage.get_projects()
85
+ synced_any = False
86
+ for project in projects:
87
+ space_id = SQLiteStorage.get_space_id(project)
88
+ if space_id and SQLiteStorage.has_pending_data(project):
89
+ sync_incremental(
90
+ project, space_id, private=args.private, pending_only=True
91
+ )
92
+ synced_any = True
93
+ if not synced_any:
94
+ print("No projects with unsynced data found.")
95
+ else:
96
+ space_id = args.space_id
97
+ if space_id is None:
98
+ space_id = SQLiteStorage.get_space_id(args.project)
99
+ sync(
100
+ project=args.project,
101
+ space_id=space_id,
102
+ private=args.private,
103
+ force=args.force,
104
+ sdk=args.sdk,
105
+ )
106
+
107
+
108
+ def _extract_reports(
109
+ run: str, logs: list[dict], report_name: str | None = None
110
+ ) -> list[dict]:
111
+ reports = []
112
+ for log in logs:
113
+ timestamp = log.get("timestamp")
114
+ step = log.get("step")
115
+ for key, value in log.items():
116
+ if report_name is not None and key != report_name:
117
+ continue
118
+ if isinstance(value, dict) and value.get("_type") == Markdown.TYPE:
119
+ content = value.get("_value")
120
+ if isinstance(content, str):
121
+ reports.append(
122
+ {
123
+ "run": run,
124
+ "report": key,
125
+ "step": step,
126
+ "timestamp": timestamp,
127
+ "content": content,
128
+ }
129
+ )
130
+ return reports
131
+
132
+
133
+ def main():
134
+ parser = argparse.ArgumentParser(description="Trackio CLI")
135
+ parser.add_argument(
136
+ "--space",
137
+ required=False,
138
+ help="HF Space ID (e.g. 'user/space') or Space URL to query remotely.",
139
+ )
140
+ parser.add_argument(
141
+ "--hf-token",
142
+ required=False,
143
+ help="HF token for accessing private Spaces.",
144
+ )
145
+ subparsers = parser.add_subparsers(dest="command")
146
+
147
+ ui_parser = subparsers.add_parser(
148
+ "show", help="Show the Trackio dashboard UI for a project"
149
+ )
150
+ ui_parser.add_argument(
151
+ "--project", required=False, help="Project name to show in the dashboard"
152
+ )
153
+ ui_parser.add_argument(
154
+ "--theme",
155
+ required=False,
156
+ default="default",
157
+ help="A Gradio Theme to use for the dashboard instead of the default, can be a built-in theme (e.g. 'soft', 'citrus'), or a theme from the Hub (e.g. 'gstaff/xkcd').",
158
+ )
159
+ ui_parser.add_argument(
160
+ "--mcp-server",
161
+ action="store_true",
162
+ help="Enable MCP server functionality. The Trackio dashboard will be set up as an MCP server and certain functions will be exposed as MCP tools.",
163
+ )
164
+ ui_parser.add_argument(
165
+ "--footer",
166
+ action="store_true",
167
+ default=True,
168
+ help="Show the Gradio footer. Use --no-footer to hide it.",
169
+ )
170
+ ui_parser.add_argument(
171
+ "--no-footer",
172
+ dest="footer",
173
+ action="store_false",
174
+ help="Hide the Gradio footer.",
175
+ )
176
+ ui_parser.add_argument(
177
+ "--color-palette",
178
+ required=False,
179
+ help="Comma-separated list of hex color codes for plot lines (e.g. '#FF0000,#00FF00,#0000FF'). If not provided, the TRACKIO_COLOR_PALETTE environment variable will be used, or the default palette if not set.",
180
+ )
181
+ ui_parser.add_argument(
182
+ "--host",
183
+ required=False,
184
+ help="Host to bind the server to (e.g. '0.0.0.0' for remote access). If not provided, defaults to '127.0.0.1' (localhost only).",
185
+ )
186
+
187
+ subparsers.add_parser(
188
+ "status",
189
+ help="Show the status of all local Trackio projects, including sync status.",
190
+ )
191
+
192
+ sync_parser = subparsers.add_parser(
193
+ "sync",
194
+ help="Sync a local project's database to a Hugging Face Space. If the Space does not exist, it will be created.",
195
+ )
196
+ sync_parser.add_argument(
197
+ "--project",
198
+ required=False,
199
+ help="The name of the local project.",
200
+ )
201
+ sync_parser.add_argument(
202
+ "--space-id",
203
+ required=False,
204
+ help="The Hugging Face Space ID where the project will be synced (e.g. username/space_id). If not provided, uses the previously-configured Space.",
205
+ )
206
+ sync_parser.add_argument(
207
+ "--all",
208
+ action="store_true",
209
+ dest="sync_all",
210
+ help="Sync all projects that have unsynced data to their configured Spaces.",
211
+ )
212
+ sync_parser.add_argument(
213
+ "--private",
214
+ action="store_true",
215
+ help="Make the Hugging Face Space private if creating a new Space. By default, the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.",
216
+ )
217
+ sync_parser.add_argument(
218
+ "--force",
219
+ action="store_true",
220
+ help="Overwrite the existing database without prompting for confirmation.",
221
+ )
222
+ sync_parser.add_argument(
223
+ "--sdk",
224
+ choices=["gradio", "static"],
225
+ default="gradio",
226
+ help="The type of Space to deploy. 'gradio' (default) deploys a live Gradio server. 'static' deploys a static Space that reads from an HF Bucket.",
227
+ )
228
+
229
+ freeze_parser = subparsers.add_parser(
230
+ "freeze",
231
+ help="Create a one-time static Space snapshot from a project's data.",
232
+ )
233
+ freeze_parser.add_argument(
234
+ "--space-id",
235
+ required=True,
236
+ help="The source Gradio Space ID (e.g. username/space_id).",
237
+ )
238
+ freeze_parser.add_argument(
239
+ "--project",
240
+ required=True,
241
+ help="The name of the project to freeze into a static snapshot.",
242
+ )
243
+ freeze_parser.add_argument(
244
+ "--new-space-id",
245
+ required=False,
246
+ help="The Space ID for the new static Space. Defaults to {space_id}_static.",
247
+ )
248
+ freeze_parser.add_argument(
249
+ "--private",
250
+ action="store_true",
251
+ help="Make the new static Space private.",
252
+ )
253
+
254
+ list_parser = subparsers.add_parser(
255
+ "list",
256
+ help="List projects, runs, or metrics",
257
+ )
258
+ list_subparsers = list_parser.add_subparsers(dest="list_type", required=True)
259
+
260
+ list_projects_parser = list_subparsers.add_parser(
261
+ "projects",
262
+ help="List all projects",
263
+ )
264
+ list_projects_parser.add_argument(
265
+ "--json",
266
+ action="store_true",
267
+ help="Output in JSON format",
268
+ )
269
+
270
+ list_runs_parser = list_subparsers.add_parser(
271
+ "runs",
272
+ help="List runs for a project",
273
+ )
274
+ list_runs_parser.add_argument(
275
+ "--project",
276
+ required=True,
277
+ help="Project name",
278
+ )
279
+ list_runs_parser.add_argument(
280
+ "--json",
281
+ action="store_true",
282
+ help="Output in JSON format",
283
+ )
284
+
285
+ list_metrics_parser = list_subparsers.add_parser(
286
+ "metrics",
287
+ help="List metrics for a run",
288
+ )
289
+ list_metrics_parser.add_argument(
290
+ "--project",
291
+ required=True,
292
+ help="Project name",
293
+ )
294
+ list_metrics_parser.add_argument(
295
+ "--run",
296
+ required=True,
297
+ help="Run name",
298
+ )
299
+ list_metrics_parser.add_argument(
300
+ "--json",
301
+ action="store_true",
302
+ help="Output in JSON format",
303
+ )
304
+
305
+ list_system_metrics_parser = list_subparsers.add_parser(
306
+ "system-metrics",
307
+ help="List system metrics for a run",
308
+ )
309
+ list_system_metrics_parser.add_argument(
310
+ "--project",
311
+ required=True,
312
+ help="Project name",
313
+ )
314
+ list_system_metrics_parser.add_argument(
315
+ "--run",
316
+ required=True,
317
+ help="Run name",
318
+ )
319
+ list_system_metrics_parser.add_argument(
320
+ "--json",
321
+ action="store_true",
322
+ help="Output in JSON format",
323
+ )
324
+
325
+ list_alerts_parser = list_subparsers.add_parser(
326
+ "alerts",
327
+ help="List alerts for a project or run",
328
+ )
329
+ list_alerts_parser.add_argument(
330
+ "--project",
331
+ required=True,
332
+ help="Project name",
333
+ )
334
+ list_alerts_parser.add_argument(
335
+ "--run",
336
+ required=False,
337
+ help="Run name (optional)",
338
+ )
339
+ list_alerts_parser.add_argument(
340
+ "--level",
341
+ required=False,
342
+ help="Filter by alert level (info, warn, error)",
343
+ )
344
+ list_alerts_parser.add_argument(
345
+ "--json",
346
+ action="store_true",
347
+ help="Output in JSON format",
348
+ )
349
+ list_alerts_parser.add_argument(
350
+ "--since",
351
+ required=False,
352
+ help="Only show alerts after this ISO 8601 timestamp",
353
+ )
354
+
355
+ list_reports_parser = list_subparsers.add_parser(
356
+ "reports",
357
+ help="List markdown reports for a project or run",
358
+ )
359
+ list_reports_parser.add_argument(
360
+ "--project",
361
+ required=True,
362
+ help="Project name",
363
+ )
364
+ list_reports_parser.add_argument(
365
+ "--run",
366
+ required=False,
367
+ help="Run name (optional)",
368
+ )
369
+ list_reports_parser.add_argument(
370
+ "--json",
371
+ action="store_true",
372
+ help="Output in JSON format",
373
+ )
374
+
375
+ get_parser = subparsers.add_parser(
376
+ "get",
377
+ help="Get project, run, or metric information",
378
+ )
379
+ get_subparsers = get_parser.add_subparsers(dest="get_type", required=True)
380
+
381
+ get_project_parser = get_subparsers.add_parser(
382
+ "project",
383
+ help="Get project summary",
384
+ )
385
+ get_project_parser.add_argument(
386
+ "--project",
387
+ required=True,
388
+ help="Project name",
389
+ )
390
+ get_project_parser.add_argument(
391
+ "--json",
392
+ action="store_true",
393
+ help="Output in JSON format",
394
+ )
395
+
396
+ get_run_parser = get_subparsers.add_parser(
397
+ "run",
398
+ help="Get run summary",
399
+ )
400
+ get_run_parser.add_argument(
401
+ "--project",
402
+ required=True,
403
+ help="Project name",
404
+ )
405
+ get_run_parser.add_argument(
406
+ "--run",
407
+ required=True,
408
+ help="Run name",
409
+ )
410
+ get_run_parser.add_argument(
411
+ "--json",
412
+ action="store_true",
413
+ help="Output in JSON format",
414
+ )
415
+
416
+ get_metric_parser = get_subparsers.add_parser(
417
+ "metric",
418
+ help="Get metric values for a run",
419
+ )
420
+ get_metric_parser.add_argument(
421
+ "--project",
422
+ required=True,
423
+ help="Project name",
424
+ )
425
+ get_metric_parser.add_argument(
426
+ "--run",
427
+ required=True,
428
+ help="Run name",
429
+ )
430
+ get_metric_parser.add_argument(
431
+ "--metric",
432
+ required=True,
433
+ help="Metric name",
434
+ )
435
+ get_metric_parser.add_argument(
436
+ "--step",
437
+ type=int,
438
+ required=False,
439
+ help="Get metric at exactly this step",
440
+ )
441
+ get_metric_parser.add_argument(
442
+ "--around",
443
+ type=int,
444
+ required=False,
445
+ help="Get metrics around this step (use with --window)",
446
+ )
447
+ get_metric_parser.add_argument(
448
+ "--at-time",
449
+ required=False,
450
+ help="Get metrics around this ISO 8601 timestamp (use with --window)",
451
+ )
452
+ get_metric_parser.add_argument(
453
+ "--window",
454
+ type=int,
455
+ required=False,
456
+ default=10,
457
+ help="Window size: ±steps for --around, ±seconds for --at-time (default: 10)",
458
+ )
459
+ get_metric_parser.add_argument(
460
+ "--json",
461
+ action="store_true",
462
+ help="Output in JSON format",
463
+ )
464
+
465
+ get_snapshot_parser = get_subparsers.add_parser(
466
+ "snapshot",
467
+ help="Get all metrics at/around a step or timestamp",
468
+ )
469
+ get_snapshot_parser.add_argument(
470
+ "--project",
471
+ required=True,
472
+ help="Project name",
473
+ )
474
+ get_snapshot_parser.add_argument(
475
+ "--run",
476
+ required=True,
477
+ help="Run name",
478
+ )
479
+ get_snapshot_parser.add_argument(
480
+ "--step",
481
+ type=int,
482
+ required=False,
483
+ help="Get all metrics at exactly this step",
484
+ )
485
+ get_snapshot_parser.add_argument(
486
+ "--around",
487
+ type=int,
488
+ required=False,
489
+ help="Get all metrics around this step (use with --window)",
490
+ )
491
+ get_snapshot_parser.add_argument(
492
+ "--at-time",
493
+ required=False,
494
+ help="Get all metrics around this ISO 8601 timestamp (use with --window)",
495
+ )
496
+ get_snapshot_parser.add_argument(
497
+ "--window",
498
+ type=int,
499
+ required=False,
500
+ default=10,
501
+ help="Window size: ±steps for --around, ±seconds for --at-time (default: 10)",
502
+ )
503
+ get_snapshot_parser.add_argument(
504
+ "--json",
505
+ action="store_true",
506
+ help="Output in JSON format",
507
+ )
508
+
509
+ get_system_metric_parser = get_subparsers.add_parser(
510
+ "system-metric",
511
+ help="Get system metric values for a run",
512
+ )
513
+ get_system_metric_parser.add_argument(
514
+ "--project",
515
+ required=True,
516
+ help="Project name",
517
+ )
518
+ get_system_metric_parser.add_argument(
519
+ "--run",
520
+ required=True,
521
+ help="Run name",
522
+ )
523
+ get_system_metric_parser.add_argument(
524
+ "--metric",
525
+ required=False,
526
+ help="System metric name (optional, if not provided returns all system metrics)",
527
+ )
528
+ get_system_metric_parser.add_argument(
529
+ "--json",
530
+ action="store_true",
531
+ help="Output in JSON format",
532
+ )
533
+
534
+ get_alerts_parser = get_subparsers.add_parser(
535
+ "alerts",
536
+ help="Get alerts for a project or run",
537
+ )
538
+ get_alerts_parser.add_argument(
539
+ "--project",
540
+ required=True,
541
+ help="Project name",
542
+ )
543
+ get_alerts_parser.add_argument(
544
+ "--run",
545
+ required=False,
546
+ help="Run name (optional)",
547
+ )
548
+ get_alerts_parser.add_argument(
549
+ "--level",
550
+ required=False,
551
+ help="Filter by alert level (info, warn, error)",
552
+ )
553
+ get_alerts_parser.add_argument(
554
+ "--json",
555
+ action="store_true",
556
+ help="Output in JSON format",
557
+ )
558
+ get_alerts_parser.add_argument(
559
+ "--since",
560
+ required=False,
561
+ help="Only show alerts after this ISO 8601 timestamp",
562
+ )
563
+
564
+ get_report_parser = get_subparsers.add_parser(
565
+ "report",
566
+ help="Get markdown report entries for a run",
567
+ )
568
+ get_report_parser.add_argument(
569
+ "--project",
570
+ required=True,
571
+ help="Project name",
572
+ )
573
+ get_report_parser.add_argument(
574
+ "--run",
575
+ required=True,
576
+ help="Run name",
577
+ )
578
+ get_report_parser.add_argument(
579
+ "--report",
580
+ required=True,
581
+ help="Report metric name",
582
+ )
583
+ get_report_parser.add_argument(
584
+ "--json",
585
+ action="store_true",
586
+ help="Output in JSON format",
587
+ )
588
+
589
+ skills_parser = subparsers.add_parser(
590
+ "skills",
591
+ help="Manage Trackio skills for AI coding assistants",
592
+ )
593
+ skills_subparsers = skills_parser.add_subparsers(
594
+ dest="skills_action", required=True
595
+ )
596
+ skills_add_parser = skills_subparsers.add_parser(
597
+ "add",
598
+ help="Download and install the Trackio skill for an AI assistant",
599
+ )
600
+ skills_add_parser.add_argument(
601
+ "--cursor",
602
+ action="store_true",
603
+ help="Install for Cursor",
604
+ )
605
+ skills_add_parser.add_argument(
606
+ "--claude",
607
+ action="store_true",
608
+ help="Install for Claude Code",
609
+ )
610
+ skills_add_parser.add_argument(
611
+ "--codex",
612
+ action="store_true",
613
+ help="Install for Codex",
614
+ )
615
+ skills_add_parser.add_argument(
616
+ "--opencode",
617
+ action="store_true",
618
+ help="Install for OpenCode",
619
+ )
620
+ skills_add_parser.add_argument(
621
+ "--global",
622
+ dest="global_",
623
+ action="store_true",
624
+ help="Install globally (user-level) instead of in the current project directory",
625
+ )
626
+ skills_add_parser.add_argument(
627
+ "--dest",
628
+ type=str,
629
+ required=False,
630
+ help="Install into a custom destination (path to skills directory)",
631
+ )
632
+ skills_add_parser.add_argument(
633
+ "--force",
634
+ action="store_true",
635
+ help="Overwrite existing skill if it already exists",
636
+ )
637
+
638
+ args, unknown_args = parser.parse_known_args()
639
+ if unknown_args:
640
+ trailing_global_parser = argparse.ArgumentParser(add_help=False)
641
+ trailing_global_parser.add_argument("--space", required=False)
642
+ trailing_global_parser.add_argument("--hf-token", required=False)
643
+ trailing_globals, remaining_unknown = trailing_global_parser.parse_known_args(
644
+ unknown_args
645
+ )
646
+ if remaining_unknown:
647
+ parser.error(f"unrecognized arguments: {' '.join(remaining_unknown)}")
648
+ if trailing_globals.space is not None:
649
+ args.space = trailing_globals.space
650
+ if trailing_globals.hf_token is not None:
651
+ args.hf_token = trailing_globals.hf_token
652
+
653
+ if args.command in ("show", "status", "sync", "freeze", "skills") and _get_space(
654
+ args
655
+ ):
656
+ error_exit(
657
+ f"The '{args.command}' command does not support --space (remote mode)."
658
+ )
659
+
660
+ if args.command == "show":
661
+ color_palette = None
662
+ if args.color_palette:
663
+ color_palette = [color.strip() for color in args.color_palette.split(",")]
664
+ show(
665
+ project=args.project,
666
+ theme=args.theme,
667
+ mcp_server=args.mcp_server,
668
+ footer=args.footer,
669
+ color_palette=color_palette,
670
+ host=args.host,
671
+ )
672
+ elif args.command == "status":
673
+ _handle_status()
674
+ elif args.command == "sync":
675
+ _handle_sync(args)
676
+ elif args.command == "freeze":
677
+ freeze(
678
+ space_id=args.space_id,
679
+ project=args.project,
680
+ new_space_id=args.new_space_id,
681
+ private=args.private,
682
+ )
683
+ elif args.command == "list":
684
+ remote = _get_remote(args)
685
+ if args.list_type == "projects":
686
+ if remote:
687
+ projects = remote.predict(api_name="/get_all_projects")
688
+ else:
689
+ projects = SQLiteStorage.get_projects()
690
+ if args.json:
691
+ print(format_json({"projects": projects}))
692
+ else:
693
+ print(format_list(projects, "Projects"))
694
+ elif args.list_type == "runs":
695
+ if remote:
696
+ runs = remote.predict(args.project, api_name="/get_runs_for_project")
697
+ else:
698
+ db_path = SQLiteStorage.get_project_db_path(args.project)
699
+ if not db_path.exists():
700
+ error_exit(f"Project '{args.project}' not found.")
701
+ runs = SQLiteStorage.get_runs(args.project)
702
+ if args.json:
703
+ print(format_json({"project": args.project, "runs": runs}))
704
+ else:
705
+ print(format_list(runs, f"Runs in '{args.project}'"))
706
+ elif args.list_type == "metrics":
707
+ if remote:
708
+ metrics = remote.predict(
709
+ args.project, args.run, api_name="/get_metrics_for_run"
710
+ )
711
+ else:
712
+ db_path = SQLiteStorage.get_project_db_path(args.project)
713
+ if not db_path.exists():
714
+ error_exit(f"Project '{args.project}' not found.")
715
+ runs = SQLiteStorage.get_runs(args.project)
716
+ if args.run not in runs:
717
+ error_exit(
718
+ f"Run '{args.run}' not found in project '{args.project}'."
719
+ )
720
+ metrics = SQLiteStorage.get_all_metrics_for_run(args.project, args.run)
721
+ if args.json:
722
+ print(
723
+ format_json(
724
+ {"project": args.project, "run": args.run, "metrics": metrics}
725
+ )
726
+ )
727
+ else:
728
+ print(
729
+ format_list(
730
+ metrics, f"Metrics for '{args.run}' in '{args.project}'"
731
+ )
732
+ )
733
+ elif args.list_type == "system-metrics":
734
+ if remote:
735
+ system_metrics = remote.predict(
736
+ args.project, args.run, api_name="/get_system_metrics_for_run"
737
+ )
738
+ else:
739
+ db_path = SQLiteStorage.get_project_db_path(args.project)
740
+ if not db_path.exists():
741
+ error_exit(f"Project '{args.project}' not found.")
742
+ runs = SQLiteStorage.get_runs(args.project)
743
+ if args.run not in runs:
744
+ error_exit(
745
+ f"Run '{args.run}' not found in project '{args.project}'."
746
+ )
747
+ system_metrics = SQLiteStorage.get_all_system_metrics_for_run(
748
+ args.project, args.run
749
+ )
750
+ if args.json:
751
+ print(
752
+ format_json(
753
+ {
754
+ "project": args.project,
755
+ "run": args.run,
756
+ "system_metrics": system_metrics,
757
+ }
758
+ )
759
+ )
760
+ else:
761
+ print(format_system_metric_names(system_metrics))
762
+ elif args.list_type == "alerts":
763
+ if remote:
764
+ alerts = remote.predict(
765
+ args.project,
766
+ args.run,
767
+ args.level,
768
+ args.since,
769
+ api_name="/get_alerts",
770
+ )
771
+ else:
772
+ db_path = SQLiteStorage.get_project_db_path(args.project)
773
+ if not db_path.exists():
774
+ error_exit(f"Project '{args.project}' not found.")
775
+ alerts = SQLiteStorage.get_alerts(
776
+ args.project,
777
+ run_name=args.run,
778
+ level=args.level,
779
+ since=args.since,
780
+ )
781
+ if args.json:
782
+ print(
783
+ format_json(
784
+ {
785
+ "project": args.project,
786
+ "run": args.run,
787
+ "level": args.level,
788
+ "since": args.since,
789
+ "alerts": alerts,
790
+ }
791
+ )
792
+ )
793
+ else:
794
+ print(format_alerts(alerts))
795
+ elif args.list_type == "reports":
796
+ if remote:
797
+ runs = remote.predict(args.project, api_name="/get_runs_for_project")
798
+ else:
799
+ db_path = SQLiteStorage.get_project_db_path(args.project)
800
+ if not db_path.exists():
801
+ error_exit(f"Project '{args.project}' not found.")
802
+ runs = SQLiteStorage.get_runs(args.project)
803
+ if args.run and args.run not in runs:
804
+ error_exit(f"Run '{args.run}' not found in project '{args.project}'.")
805
+
806
+ target_runs = [args.run] if args.run else runs
807
+ all_reports = []
808
+ for run_name in target_runs:
809
+ if remote:
810
+ logs = remote.predict(args.project, run_name, api_name="/get_logs")
811
+ else:
812
+ logs = SQLiteStorage.get_logs(args.project, run_name)
813
+ all_reports.extend(_extract_reports(run_name, logs))
814
+
815
+ if args.json:
816
+ print(
817
+ format_json(
818
+ {
819
+ "project": args.project,
820
+ "run": args.run,
821
+ "reports": all_reports,
822
+ }
823
+ )
824
+ )
825
+ else:
826
+ report_lines = [
827
+ f"{entry['run']} | {entry['report']} | step={entry['step']} | {entry['timestamp']}"
828
+ for entry in all_reports
829
+ ]
830
+ if args.run:
831
+ print(
832
+ format_list(
833
+ report_lines,
834
+ f"Reports for '{args.run}' in '{args.project}'",
835
+ )
836
+ )
837
+ else:
838
+ print(format_list(report_lines, f"Reports in '{args.project}'"))
839
+ elif args.command == "get":
840
+ remote = _get_remote(args)
841
+ if args.get_type == "project":
842
+ if remote:
843
+ summary = remote.predict(args.project, api_name="/get_project_summary")
844
+ else:
845
+ db_path = SQLiteStorage.get_project_db_path(args.project)
846
+ if not db_path.exists():
847
+ error_exit(f"Project '{args.project}' not found.")
848
+ summary = get_project_summary(args.project)
849
+ if args.json:
850
+ print(format_json(summary))
851
+ else:
852
+ print(format_project_summary(summary))
853
+ elif args.get_type == "run":
854
+ if remote:
855
+ summary = remote.predict(
856
+ args.project, args.run, api_name="/get_run_summary"
857
+ )
858
+ else:
859
+ db_path = SQLiteStorage.get_project_db_path(args.project)
860
+ if not db_path.exists():
861
+ error_exit(f"Project '{args.project}' not found.")
862
+ runs = SQLiteStorage.get_runs(args.project)
863
+ if args.run not in runs:
864
+ error_exit(
865
+ f"Run '{args.run}' not found in project '{args.project}'."
866
+ )
867
+ summary = get_run_summary(args.project, args.run)
868
+ if args.json:
869
+ print(format_json(summary))
870
+ else:
871
+ print(format_run_summary(summary))
872
+ elif args.get_type == "metric":
873
+ at_time = getattr(args, "at_time", None)
874
+ if remote:
875
+ values = remote.predict(
876
+ args.project,
877
+ args.run,
878
+ args.metric,
879
+ args.step,
880
+ args.around,
881
+ at_time,
882
+ args.window,
883
+ api_name="/get_metric_values",
884
+ )
885
+ else:
886
+ db_path = SQLiteStorage.get_project_db_path(args.project)
887
+ if not db_path.exists():
888
+ error_exit(f"Project '{args.project}' not found.")
889
+ runs = SQLiteStorage.get_runs(args.project)
890
+ if args.run not in runs:
891
+ error_exit(
892
+ f"Run '{args.run}' not found in project '{args.project}'."
893
+ )
894
+ metrics = SQLiteStorage.get_all_metrics_for_run(args.project, args.run)
895
+ if args.metric not in metrics:
896
+ error_exit(
897
+ f"Metric '{args.metric}' not found in run '{args.run}' of project '{args.project}'."
898
+ )
899
+ values = SQLiteStorage.get_metric_values(
900
+ args.project,
901
+ args.run,
902
+ args.metric,
903
+ step=args.step,
904
+ around_step=args.around,
905
+ at_time=at_time,
906
+ window=args.window,
907
+ )
908
+ if args.json:
909
+ print(
910
+ format_json(
911
+ {
912
+ "project": args.project,
913
+ "run": args.run,
914
+ "metric": args.metric,
915
+ "values": values,
916
+ }
917
+ )
918
+ )
919
+ else:
920
+ print(format_metric_values(values))
921
+ elif args.get_type == "snapshot":
922
+ if not args.step and not args.around and not getattr(args, "at_time", None):
923
+ error_exit(
924
+ "Provide --step, --around (with --window), or --at-time (with --window)."
925
+ )
926
+ at_time = getattr(args, "at_time", None)
927
+ if remote:
928
+ snapshot = remote.predict(
929
+ args.project,
930
+ args.run,
931
+ args.step,
932
+ args.around,
933
+ at_time,
934
+ args.window,
935
+ api_name="/get_snapshot",
936
+ )
937
+ else:
938
+ db_path = SQLiteStorage.get_project_db_path(args.project)
939
+ if not db_path.exists():
940
+ error_exit(f"Project '{args.project}' not found.")
941
+ runs = SQLiteStorage.get_runs(args.project)
942
+ if args.run not in runs:
943
+ error_exit(
944
+ f"Run '{args.run}' not found in project '{args.project}'."
945
+ )
946
+ snapshot = SQLiteStorage.get_snapshot(
947
+ args.project,
948
+ args.run,
949
+ step=args.step,
950
+ around_step=args.around,
951
+ at_time=at_time,
952
+ window=args.window,
953
+ )
954
+ if args.json:
955
+ result = {
956
+ "project": args.project,
957
+ "run": args.run,
958
+ "metrics": snapshot,
959
+ }
960
+ if args.step is not None:
961
+ result["step"] = args.step
962
+ if args.around is not None:
963
+ result["around"] = args.around
964
+ result["window"] = args.window
965
+ if at_time is not None:
966
+ result["at_time"] = at_time
967
+ result["window"] = args.window
968
+ print(format_json(result))
969
+ else:
970
+ print(format_snapshot(snapshot))
971
+ elif args.get_type == "system-metric":
972
+ if remote:
973
+ system_metrics = remote.predict(
974
+ args.project, args.run, api_name="/get_system_logs"
975
+ )
976
+ if args.metric:
977
+ all_system_metric_names = remote.predict(
978
+ args.project,
979
+ args.run,
980
+ api_name="/get_system_metrics_for_run",
981
+ )
982
+ if args.metric not in all_system_metric_names:
983
+ error_exit(
984
+ f"System metric '{args.metric}' not found in run '{args.run}' of project '{args.project}'."
985
+ )
986
+ filtered_metrics = [
987
+ {
988
+ k: v
989
+ for k, v in entry.items()
990
+ if k == "timestamp" or k == args.metric
991
+ }
992
+ for entry in system_metrics
993
+ if args.metric in entry
994
+ ]
995
+ if args.json:
996
+ print(
997
+ format_json(
998
+ {
999
+ "project": args.project,
1000
+ "run": args.run,
1001
+ "metric": args.metric,
1002
+ "values": filtered_metrics,
1003
+ }
1004
+ )
1005
+ )
1006
+ else:
1007
+ print(format_system_metrics(filtered_metrics))
1008
+ else:
1009
+ if args.json:
1010
+ print(
1011
+ format_json(
1012
+ {
1013
+ "project": args.project,
1014
+ "run": args.run,
1015
+ "system_metrics": system_metrics,
1016
+ }
1017
+ )
1018
+ )
1019
+ else:
1020
+ print(format_system_metrics(system_metrics))
1021
+ else:
1022
+ db_path = SQLiteStorage.get_project_db_path(args.project)
1023
+ if not db_path.exists():
1024
+ error_exit(f"Project '{args.project}' not found.")
1025
+ runs = SQLiteStorage.get_runs(args.project)
1026
+ if args.run not in runs:
1027
+ error_exit(
1028
+ f"Run '{args.run}' not found in project '{args.project}'."
1029
+ )
1030
+ if args.metric:
1031
+ system_metrics = SQLiteStorage.get_system_logs(
1032
+ args.project, args.run
1033
+ )
1034
+ all_system_metric_names = (
1035
+ SQLiteStorage.get_all_system_metrics_for_run(
1036
+ args.project, args.run
1037
+ )
1038
+ )
1039
+ if args.metric not in all_system_metric_names:
1040
+ error_exit(
1041
+ f"System metric '{args.metric}' not found in run '{args.run}' of project '{args.project}'."
1042
+ )
1043
+ filtered_metrics = [
1044
+ {
1045
+ k: v
1046
+ for k, v in entry.items()
1047
+ if k == "timestamp" or k == args.metric
1048
+ }
1049
+ for entry in system_metrics
1050
+ if args.metric in entry
1051
+ ]
1052
+ if args.json:
1053
+ print(
1054
+ format_json(
1055
+ {
1056
+ "project": args.project,
1057
+ "run": args.run,
1058
+ "metric": args.metric,
1059
+ "values": filtered_metrics,
1060
+ }
1061
+ )
1062
+ )
1063
+ else:
1064
+ print(format_system_metrics(filtered_metrics))
1065
+ else:
1066
+ system_metrics = SQLiteStorage.get_system_logs(
1067
+ args.project, args.run
1068
+ )
1069
+ if args.json:
1070
+ print(
1071
+ format_json(
1072
+ {
1073
+ "project": args.project,
1074
+ "run": args.run,
1075
+ "system_metrics": system_metrics,
1076
+ }
1077
+ )
1078
+ )
1079
+ else:
1080
+ print(format_system_metrics(system_metrics))
1081
+ elif args.get_type == "alerts":
1082
+ if remote:
1083
+ alerts = remote.predict(
1084
+ args.project,
1085
+ args.run,
1086
+ args.level,
1087
+ args.since,
1088
+ api_name="/get_alerts",
1089
+ )
1090
+ else:
1091
+ db_path = SQLiteStorage.get_project_db_path(args.project)
1092
+ if not db_path.exists():
1093
+ error_exit(f"Project '{args.project}' not found.")
1094
+ alerts = SQLiteStorage.get_alerts(
1095
+ args.project,
1096
+ run_name=args.run,
1097
+ level=args.level,
1098
+ since=args.since,
1099
+ )
1100
+ if args.json:
1101
+ print(
1102
+ format_json(
1103
+ {
1104
+ "project": args.project,
1105
+ "run": args.run,
1106
+ "level": args.level,
1107
+ "since": args.since,
1108
+ "alerts": alerts,
1109
+ }
1110
+ )
1111
+ )
1112
+ else:
1113
+ print(format_alerts(alerts))
1114
+ elif args.get_type == "report":
1115
+ if remote:
1116
+ logs = remote.predict(args.project, args.run, api_name="/get_logs")
1117
+ else:
1118
+ db_path = SQLiteStorage.get_project_db_path(args.project)
1119
+ if not db_path.exists():
1120
+ error_exit(f"Project '{args.project}' not found.")
1121
+ runs = SQLiteStorage.get_runs(args.project)
1122
+ if args.run not in runs:
1123
+ error_exit(
1124
+ f"Run '{args.run}' not found in project '{args.project}'."
1125
+ )
1126
+ logs = SQLiteStorage.get_logs(args.project, args.run)
1127
+
1128
+ reports = _extract_reports(args.run, logs, report_name=args.report)
1129
+ if not reports:
1130
+ error_exit(
1131
+ f"Report '{args.report}' not found in run '{args.run}' of project '{args.project}'."
1132
+ )
1133
+
1134
+ if args.json:
1135
+ print(
1136
+ format_json(
1137
+ {
1138
+ "project": args.project,
1139
+ "run": args.run,
1140
+ "report": args.report,
1141
+ "values": reports,
1142
+ }
1143
+ )
1144
+ )
1145
+ else:
1146
+ output = []
1147
+ for idx, entry in enumerate(reports, start=1):
1148
+ output.append(
1149
+ f"Entry {idx} | step={entry['step']} | timestamp={entry['timestamp']}"
1150
+ )
1151
+ output.append(entry["content"])
1152
+ if idx < len(reports):
1153
+ output.append("-" * 80)
1154
+ print("\n".join(output))
1155
+ elif args.command == "skills":
1156
+ if args.skills_action == "add":
1157
+ _handle_skills_add(args)
1158
+ else:
1159
+ parser.print_help()
1160
+
1161
+
1162
+ def _handle_skills_add(args):
1163
+ import shutil
1164
+ from pathlib import Path
1165
+
1166
+ try:
1167
+ from huggingface_hub.cli.skills import (
1168
+ CENTRAL_GLOBAL,
1169
+ CENTRAL_LOCAL,
1170
+ GLOBAL_TARGETS,
1171
+ LOCAL_TARGETS,
1172
+ )
1173
+ except (ImportError, ModuleNotFoundError):
1174
+ error_exit(
1175
+ "The 'trackio skills' command requires huggingface_hub >= 1.4.0.\n"
1176
+ "Please upgrade: pip install --upgrade huggingface_hub"
1177
+ )
1178
+
1179
+ SKILL_ID = "trackio"
1180
+ GITHUB_RAW = "https://raw.githubusercontent.com/gradio-app/trackio/main"
1181
+ SKILL_PREFIX = ".agents/skills/trackio"
1182
+ SKILL_FILES = [
1183
+ "SKILL.md",
1184
+ "alerts.md",
1185
+ "logging_metrics.md",
1186
+ "retrieving_metrics.md",
1187
+ ]
1188
+
1189
+ if not (args.cursor or args.claude or args.codex or args.opencode or args.dest):
1190
+ error_exit(
1191
+ "Pick a destination via --cursor, --claude, --codex, --opencode, or --dest."
1192
+ )
1193
+
1194
+ def download(url: str) -> str:
1195
+ from huggingface_hub.utils import get_session
1196
+
1197
+ try:
1198
+ response = get_session().get(url)
1199
+ response.raise_for_status()
1200
+ except Exception as e:
1201
+ error_exit(
1202
+ f"Failed to download {url}\n{e}\n\n"
1203
+ "Make sure you have internet access. The skill files are fetched from "
1204
+ "the Trackio GitHub repository."
1205
+ )
1206
+ return response.text
1207
+
1208
+ def remove_existing(path: Path, force: bool):
1209
+ if not (path.exists() or path.is_symlink()):
1210
+ return
1211
+ if not force:
1212
+ error_exit(
1213
+ f"Skill already exists at {path}.\nRe-run with --force to overwrite."
1214
+ )
1215
+ if path.is_dir() and not path.is_symlink():
1216
+ shutil.rmtree(path)
1217
+ else:
1218
+ path.unlink()
1219
+
1220
+ def install_to(skills_dir: Path, force: bool) -> Path:
1221
+ skills_dir = skills_dir.expanduser().resolve()
1222
+ skills_dir.mkdir(parents=True, exist_ok=True)
1223
+ dest = skills_dir / SKILL_ID
1224
+ remove_existing(dest, force)
1225
+ dest.mkdir()
1226
+ for fname in SKILL_FILES:
1227
+ content = download(f"{GITHUB_RAW}/{SKILL_PREFIX}/{fname}")
1228
+ (dest / fname).write_text(content, encoding="utf-8")
1229
+ return dest
1230
+
1231
+ def create_symlink(
1232
+ agent_skills_dir: Path, central_skill_path: Path, force: bool
1233
+ ) -> Path:
1234
+ agent_skills_dir = agent_skills_dir.expanduser().resolve()
1235
+ agent_skills_dir.mkdir(parents=True, exist_ok=True)
1236
+ link_path = agent_skills_dir / SKILL_ID
1237
+ remove_existing(link_path, force)
1238
+ link_path.symlink_to(os.path.relpath(central_skill_path, agent_skills_dir))
1239
+ return link_path
1240
+
1241
+ global_targets = {**GLOBAL_TARGETS, "cursor": Path("~/.cursor/skills")}
1242
+ local_targets = {**LOCAL_TARGETS, "cursor": Path(".cursor/skills")}
1243
+ targets_dict = global_targets if args.global_ else local_targets
1244
+
1245
+ if args.dest:
1246
+ if args.cursor or args.claude or args.codex or args.opencode or args.global_:
1247
+ error_exit("--dest cannot be combined with agent flags or --global.")
1248
+ skill_dest = install_to(Path(args.dest), args.force)
1249
+ print(f"Installed '{SKILL_ID}' to {skill_dest}")
1250
+ return
1251
+
1252
+ agent_targets = []
1253
+ if args.cursor:
1254
+ agent_targets.append(targets_dict["cursor"])
1255
+ if args.claude:
1256
+ agent_targets.append(targets_dict["claude"])
1257
+ if args.codex:
1258
+ agent_targets.append(targets_dict["codex"])
1259
+ if args.opencode:
1260
+ agent_targets.append(targets_dict["opencode"])
1261
+
1262
+ central_path = CENTRAL_GLOBAL if args.global_ else CENTRAL_LOCAL
1263
+ central_skill_path = install_to(central_path, args.force)
1264
+ print(f"Installed '{SKILL_ID}' to central location: {central_skill_path}")
1265
+
1266
+ for agent_target in agent_targets:
1267
+ link_path = create_symlink(agent_target, central_skill_path, args.force)
1268
+ print(f"Created symlink: {link_path}")
1269
+
1270
+
1271
+ if __name__ == "__main__":
1272
+ main()
trackio/cli_helpers.py ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import sys
3
+ from typing import Any
4
+
5
+
6
+ def format_json(data: Any) -> str:
7
+ """Format data as JSON."""
8
+ return json.dumps(data, indent=2)
9
+
10
+
11
+ def format_list(items: list[str], title: str | None = None) -> str:
12
+ """Format a list of items in human-readable format."""
13
+ if not items:
14
+ return f"No {title.lower() if title else 'items'} found."
15
+
16
+ output = []
17
+ if title:
18
+ output.append(f"{title}:")
19
+
20
+ for item in items:
21
+ output.append(f" - {item}")
22
+
23
+ return "\n".join(output)
24
+
25
+
26
+ def format_project_summary(summary: dict) -> str:
27
+ """Format project summary in human-readable format."""
28
+ output = [f"Project: {summary['project']}"]
29
+ output.append(f"Number of runs: {summary['num_runs']}")
30
+
31
+ if summary["runs"]:
32
+ output.append("\nRuns:")
33
+ for run in summary["runs"]:
34
+ output.append(f" - {run}")
35
+ else:
36
+ output.append("\nNo runs found.")
37
+
38
+ if summary.get("last_activity"):
39
+ output.append(f"\nLast activity (max step): {summary['last_activity']}")
40
+
41
+ return "\n".join(output)
42
+
43
+
44
+ def format_run_summary(summary: dict) -> str:
45
+ """Format run summary in human-readable format."""
46
+ output = [f"Project: {summary['project']}"]
47
+ output.append(f"Run: {summary['run']}")
48
+ output.append(f"Number of logs: {summary['num_logs']}")
49
+
50
+ if summary.get("last_step") is not None:
51
+ output.append(f"Last step: {summary['last_step']}")
52
+
53
+ if summary.get("metrics"):
54
+ output.append("\nMetrics:")
55
+ for metric in summary["metrics"]:
56
+ output.append(f" - {metric}")
57
+ else:
58
+ output.append("\nNo metrics found.")
59
+
60
+ config = summary.get("config")
61
+ if config:
62
+ output.append("\nConfig:")
63
+ config_display = {k: v for k, v in config.items() if not k.startswith("_")}
64
+ if config_display:
65
+ for key, value in config_display.items():
66
+ output.append(f" {key}: {value}")
67
+ else:
68
+ output.append(" (no config)")
69
+ else:
70
+ output.append("\nConfig: (no config)")
71
+
72
+ return "\n".join(output)
73
+
74
+
75
+ def format_metric_values(values: list[dict]) -> str:
76
+ """Format metric values in human-readable format."""
77
+ if not values:
78
+ return "No metric values found."
79
+
80
+ output = [f"Found {len(values)} value(s):\n"]
81
+ output.append("Step | Timestamp | Value")
82
+ output.append("-" * 50)
83
+
84
+ for value in values:
85
+ step = value.get("step", "N/A")
86
+ timestamp = value.get("timestamp", "N/A")
87
+ val = value.get("value", "N/A")
88
+ output.append(f"{step} | {timestamp} | {val}")
89
+
90
+ return "\n".join(output)
91
+
92
+
93
+ def format_system_metrics(metrics: list[dict]) -> str:
94
+ """Format system metrics in human-readable format."""
95
+ if not metrics:
96
+ return "No system metrics found."
97
+
98
+ output = [f"Found {len(metrics)} system metric entry/entries:\n"]
99
+
100
+ for i, entry in enumerate(metrics):
101
+ timestamp = entry.get("timestamp", "N/A")
102
+ output.append(f"\nEntry {i + 1} (Timestamp: {timestamp}):")
103
+ for key, value in entry.items():
104
+ if key != "timestamp":
105
+ output.append(f" {key}: {value}")
106
+
107
+ return "\n".join(output)
108
+
109
+
110
+ def format_system_metric_names(names: list[str]) -> str:
111
+ """Format system metric names in human-readable format."""
112
+ return format_list(names, "System Metrics")
113
+
114
+
115
+ def format_snapshot(snapshot: dict[str, list[dict]]) -> str:
116
+ """Format a metrics snapshot in human-readable format."""
117
+ if not snapshot:
118
+ return "No metrics found in the specified range."
119
+
120
+ output = []
121
+ for metric_name, values in sorted(snapshot.items()):
122
+ output.append(f"\n{metric_name}:")
123
+ output.append(" Step | Timestamp | Value")
124
+ output.append(" " + "-" * 48)
125
+ for v in values:
126
+ step = v.get("step", "N/A")
127
+ ts = v.get("timestamp", "N/A")
128
+ val = v.get("value", "N/A")
129
+ output.append(f" {step} | {ts} | {val}")
130
+
131
+ return "\n".join(output)
132
+
133
+
134
+ def format_alerts(alerts: list[dict]) -> str:
135
+ """Format alerts in human-readable format."""
136
+ if not alerts:
137
+ return "No alerts found."
138
+
139
+ output = [f"Found {len(alerts)} alert(s):\n"]
140
+ output.append("Timestamp | Run | Level | Title | Text | Step")
141
+ output.append("-" * 80)
142
+
143
+ for a in alerts:
144
+ ts = a.get("timestamp", "N/A")
145
+ run = a.get("run", "N/A")
146
+ level = a.get("level", "N/A").upper()
147
+ title = a.get("title", "")
148
+ text = a.get("text", "") or ""
149
+ step = a.get("step", "N/A")
150
+ output.append(f"{ts} | {run} | {level} | {title} | {text} | {step}")
151
+
152
+ return "\n".join(output)
153
+
154
+
155
+ def error_exit(message: str, code: int = 1) -> None:
156
+ """Print error message and exit."""
157
+ print(f"Error: {message}", file=sys.stderr)
158
+ sys.exit(code)
trackio/commit_scheduler.py ADDED
@@ -0,0 +1,310 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Originally copied from https://github.com/huggingface/huggingface_hub/blob/d0a948fc2a32ed6e557042a95ef3e4af97ec4a7c/src/huggingface_hub/_commit_scheduler.py
2
+
3
+ import atexit
4
+ import logging
5
+ import time
6
+ from concurrent.futures import Future
7
+ from dataclasses import dataclass
8
+ from pathlib import Path
9
+ from threading import Lock, Thread
10
+ from typing import Callable, Dict, List, Union
11
+
12
+ from huggingface_hub.hf_api import (
13
+ DEFAULT_IGNORE_PATTERNS,
14
+ CommitInfo,
15
+ CommitOperationAdd,
16
+ HfApi,
17
+ )
18
+ from huggingface_hub.utils import filter_repo_objects
19
+
20
+ logger = logging.getLogger(__name__)
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class _FileToUpload:
25
+ """Temporary dataclass to store info about files to upload. Not meant to be used directly."""
26
+
27
+ local_path: Path
28
+ path_in_repo: str
29
+ size_limit: int
30
+ last_modified: float
31
+
32
+
33
+ class CommitScheduler:
34
+ """
35
+ Scheduler to upload a local folder to the Hub at regular intervals (e.g. push to hub every 5 minutes).
36
+
37
+ The recommended way to use the scheduler is to use it as a context manager. This ensures that the scheduler is
38
+ properly stopped and the last commit is triggered when the script ends. The scheduler can also be stopped manually
39
+ with the `stop` method. Checkout the [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#scheduled-uploads)
40
+ to learn more about how to use it.
41
+
42
+ Args:
43
+ repo_id (`str`):
44
+ The id of the repo to commit to.
45
+ folder_path (`str` or `Path`):
46
+ Path to the local folder to upload regularly.
47
+ every (`int` or `float`, *optional*):
48
+ The number of minutes between each commit. Defaults to 5 minutes.
49
+ path_in_repo (`str`, *optional*):
50
+ Relative path of the directory in the repo, for example: `"checkpoints/"`. Defaults to the root folder
51
+ of the repository.
52
+ repo_type (`str`, *optional*):
53
+ The type of the repo to commit to. Defaults to `model`.
54
+ revision (`str`, *optional*):
55
+ The revision of the repo to commit to. Defaults to `main`.
56
+ private (`bool`, *optional*):
57
+ Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.
58
+ token (`str`, *optional*):
59
+ The token to use to commit to the repo. Defaults to the token saved on the machine.
60
+ allow_patterns (`List[str]` or `str`, *optional*):
61
+ If provided, only files matching at least one pattern are uploaded.
62
+ ignore_patterns (`List[str]` or `str`, *optional*):
63
+ If provided, files matching any of the patterns are not uploaded.
64
+ squash_history (`bool`, *optional*):
65
+ Whether to squash the history of the repo after each commit. Defaults to `False`. Squashing commits is
66
+ useful to avoid degraded performances on the repo when it grows too large.
67
+ hf_api (`HfApi`, *optional*):
68
+ The [`HfApi`] client to use to commit to the Hub. Can be set with custom settings (user agent, token,...).
69
+ on_before_commit (`Callable[[], None]`, *optional*):
70
+ If specified, a function that will be called before the CommitScheduler lists files to create a commit.
71
+
72
+ Example:
73
+ ```py
74
+ >>> from pathlib import Path
75
+ >>> from huggingface_hub import CommitScheduler
76
+
77
+ # Scheduler uploads every 10 minutes
78
+ >>> csv_path = Path("watched_folder/data.csv")
79
+ >>> CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path=csv_path.parent, every=10)
80
+
81
+ >>> with csv_path.open("a") as f:
82
+ ... f.write("first line")
83
+
84
+ # Some time later (...)
85
+ >>> with csv_path.open("a") as f:
86
+ ... f.write("second line")
87
+ ```
88
+
89
+ Example using a context manager:
90
+ ```py
91
+ >>> from pathlib import Path
92
+ >>> from huggingface_hub import CommitScheduler
93
+
94
+ >>> with CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path="watched_folder", every=10) as scheduler:
95
+ ... csv_path = Path("watched_folder/data.csv")
96
+ ... with csv_path.open("a") as f:
97
+ ... f.write("first line")
98
+ ... (...)
99
+ ... with csv_path.open("a") as f:
100
+ ... f.write("second line")
101
+
102
+ # Scheduler is now stopped and last commit have been triggered
103
+ ```
104
+ """
105
+
106
+ def __init__(
107
+ self,
108
+ *,
109
+ repo_id: str,
110
+ folder_path: Union[str, Path],
111
+ every: Union[int, float] = 5,
112
+ path_in_repo: str | None = None,
113
+ repo_type: str | None = None,
114
+ revision: str | None = None,
115
+ private: bool | None = None,
116
+ token: str | None = None,
117
+ allow_patterns: list[str] | str | None = None,
118
+ ignore_patterns: list[str] | str | None = None,
119
+ squash_history: bool = False,
120
+ hf_api: HfApi | None = None,
121
+ on_before_commit: Callable[[], None] | None = None,
122
+ ) -> None:
123
+ self.api = hf_api or HfApi(token=token)
124
+ self.on_before_commit = on_before_commit
125
+
126
+ # Folder
127
+ self.folder_path = Path(folder_path).expanduser().resolve()
128
+ self.path_in_repo = path_in_repo or ""
129
+ self.allow_patterns = allow_patterns
130
+
131
+ if ignore_patterns is None:
132
+ ignore_patterns = []
133
+ elif isinstance(ignore_patterns, str):
134
+ ignore_patterns = [ignore_patterns]
135
+ self.ignore_patterns = ignore_patterns + DEFAULT_IGNORE_PATTERNS
136
+
137
+ if self.folder_path.is_file():
138
+ raise ValueError(
139
+ f"'folder_path' must be a directory, not a file: '{self.folder_path}'."
140
+ )
141
+ self.folder_path.mkdir(parents=True, exist_ok=True)
142
+
143
+ # Repository
144
+ repo_url = self.api.create_repo(
145
+ repo_id=repo_id, private=private, repo_type=repo_type, exist_ok=True
146
+ )
147
+ self.repo_id = repo_url.repo_id
148
+ self.repo_type = repo_type
149
+ self.revision = revision
150
+ self.token = token
151
+
152
+ self.last_uploaded: Dict[Path, float] = {}
153
+ self.last_push_time: float | None = None
154
+
155
+ if not every > 0:
156
+ raise ValueError(f"'every' must be a positive integer, not '{every}'.")
157
+ self.lock = Lock()
158
+ self.every = every
159
+ self.squash_history = squash_history
160
+
161
+ logger.info(
162
+ f"Scheduled job to push '{self.folder_path}' to '{self.repo_id}' every {self.every} minutes."
163
+ )
164
+ self._scheduler_thread = Thread(target=self._run_scheduler, daemon=True)
165
+ self._scheduler_thread.start()
166
+ atexit.register(self._push_to_hub)
167
+
168
+ self.__stopped = False
169
+
170
+ def stop(self) -> None:
171
+ """Stop the scheduler.
172
+
173
+ A stopped scheduler cannot be restarted. Mostly for tests purposes.
174
+ """
175
+ self.__stopped = True
176
+
177
+ def __enter__(self) -> "CommitScheduler":
178
+ return self
179
+
180
+ def __exit__(self, exc_type, exc_value, traceback) -> None:
181
+ # Upload last changes before exiting
182
+ self.trigger().result()
183
+ self.stop()
184
+ return
185
+
186
+ def _run_scheduler(self) -> None:
187
+ """Dumb thread waiting between each scheduled push to Hub."""
188
+ while True:
189
+ self.last_future = self.trigger()
190
+ time.sleep(self.every * 60)
191
+ if self.__stopped:
192
+ break
193
+
194
+ def trigger(self) -> Future:
195
+ """Trigger a `push_to_hub` and return a future.
196
+
197
+ This method is automatically called every `every` minutes. You can also call it manually to trigger a commit
198
+ immediately, without waiting for the next scheduled commit.
199
+ """
200
+ return self.api.run_as_future(self._push_to_hub)
201
+
202
+ def _push_to_hub(self) -> CommitInfo | None:
203
+ if self.__stopped: # If stopped, already scheduled commits are ignored
204
+ return None
205
+
206
+ logger.info("(Background) scheduled commit triggered.")
207
+ try:
208
+ value = self.push_to_hub()
209
+ if self.squash_history:
210
+ logger.info("(Background) squashing repo history.")
211
+ self.api.super_squash_history(
212
+ repo_id=self.repo_id, repo_type=self.repo_type, branch=self.revision
213
+ )
214
+ return value
215
+ except Exception as e:
216
+ logger.error(
217
+ f"Error while pushing to Hub: {e}"
218
+ ) # Depending on the setup, error might be silenced
219
+ raise
220
+
221
+ def push_to_hub(self) -> CommitInfo | None:
222
+ """
223
+ Push folder to the Hub and return the commit info.
224
+
225
+ <Tip warning={true}>
226
+
227
+ This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
228
+ queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
229
+ issues.
230
+
231
+ </Tip>
232
+
233
+ The default behavior of `push_to_hub` is to assume an append-only folder. It lists all files in the folder and
234
+ uploads only changed files. If no changes are found, the method returns without committing anything. If you want
235
+ to change this behavior, you can inherit from [`CommitScheduler`] and override this method. This can be useful
236
+ for example to compress data together in a single file before committing. For more details and examples, check
237
+ out our [integration guide](https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#scheduled-uploads).
238
+ """
239
+ # Check files to upload (with lock)
240
+ with self.lock:
241
+ if self.on_before_commit is not None:
242
+ self.on_before_commit()
243
+
244
+ logger.debug("Listing files to upload for scheduled commit.")
245
+
246
+ # List files from folder (taken from `_prepare_upload_folder_additions`)
247
+ relpath_to_abspath = {
248
+ path.relative_to(self.folder_path).as_posix(): path
249
+ for path in sorted(
250
+ self.folder_path.glob("**/*")
251
+ ) # sorted to be deterministic
252
+ if path.is_file()
253
+ }
254
+ prefix = f"{self.path_in_repo.strip('/')}/" if self.path_in_repo else ""
255
+
256
+ # Filter with pattern + filter out unchanged files + retrieve current file size
257
+ files_to_upload: List[_FileToUpload] = []
258
+ for relpath in filter_repo_objects(
259
+ relpath_to_abspath.keys(),
260
+ allow_patterns=self.allow_patterns,
261
+ ignore_patterns=self.ignore_patterns,
262
+ ):
263
+ local_path = relpath_to_abspath[relpath]
264
+ stat = local_path.stat()
265
+ if (
266
+ self.last_uploaded.get(local_path) is None
267
+ or self.last_uploaded[local_path] != stat.st_mtime
268
+ ):
269
+ files_to_upload.append(
270
+ _FileToUpload(
271
+ local_path=local_path,
272
+ path_in_repo=prefix + relpath,
273
+ size_limit=stat.st_size,
274
+ last_modified=stat.st_mtime,
275
+ )
276
+ )
277
+
278
+ # Return if nothing to upload
279
+ if len(files_to_upload) == 0:
280
+ logger.debug("Dropping schedule commit: no changed file to upload.")
281
+ return None
282
+
283
+ # Convert `_FileToUpload` as `CommitOperationAdd` (=> compute file shas + limit to file size)
284
+ logger.debug("Removing unchanged files since previous scheduled commit.")
285
+ add_operations = [
286
+ CommitOperationAdd(
287
+ # TODO: Cap the file to its current size, even if the user append data to it while a scheduled commit is happening
288
+ # (requires an upstream fix for XET-535: `hf_xet` should support `BinaryIO` for upload)
289
+ path_or_fileobj=file_to_upload.local_path,
290
+ path_in_repo=file_to_upload.path_in_repo,
291
+ )
292
+ for file_to_upload in files_to_upload
293
+ ]
294
+
295
+ # Upload files (append mode expected - no need for lock)
296
+ logger.debug("Uploading files for scheduled commit.")
297
+ commit_info = self.api.create_commit(
298
+ repo_id=self.repo_id,
299
+ repo_type=self.repo_type,
300
+ operations=add_operations,
301
+ commit_message="Scheduled Commit",
302
+ revision=self.revision,
303
+ )
304
+
305
+ for file in files_to_upload:
306
+ self.last_uploaded[file.local_path] = file.last_modified
307
+
308
+ self.last_push_time = time.time()
309
+
310
+ return commit_info
trackio/context_vars.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import contextvars
2
+ from typing import TYPE_CHECKING
3
+
4
+ if TYPE_CHECKING:
5
+ from trackio.run import Run
6
+
7
+ current_run: contextvars.ContextVar["Run | None"] = contextvars.ContextVar(
8
+ "current_run", default=None
9
+ )
10
+ current_project: contextvars.ContextVar[str | None] = contextvars.ContextVar(
11
+ "current_project", default=None
12
+ )
13
+ current_server: contextvars.ContextVar[str | None] = contextvars.ContextVar(
14
+ "current_server", default=None
15
+ )
16
+ current_space_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
17
+ "current_space_id", default=None
18
+ )
trackio/deploy.py ADDED
@@ -0,0 +1,1004 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import importlib.metadata
2
+ import io
3
+ import json as json_mod
4
+ import os
5
+ import shutil
6
+ import sys
7
+ import tempfile
8
+ import threading
9
+ import time
10
+ from collections import Counter
11
+ from importlib.resources import files
12
+ from pathlib import Path
13
+
14
+ if sys.version_info >= (3, 11):
15
+ import tomllib
16
+ else:
17
+ import tomli as tomllib
18
+
19
+ import httpx
20
+ import huggingface_hub
21
+ from gradio_client import handle_file
22
+ from httpx import ReadTimeout
23
+ from huggingface_hub import Volume
24
+ from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError
25
+
26
+ import trackio
27
+ from trackio.bucket_storage import (
28
+ create_bucket_if_not_exists,
29
+ export_from_bucket_for_static,
30
+ upload_project_to_bucket,
31
+ upload_project_to_bucket_for_static,
32
+ )
33
+ from trackio.remote_client import RemoteClient
34
+ from trackio.sqlite_storage import SQLiteStorage
35
+ from trackio.utils import (
36
+ MEDIA_DIR,
37
+ get_or_create_project_hash,
38
+ preprocess_space_and_dataset_ids,
39
+ )
40
+
41
+ SPACE_HOST_URL = "https://{user_name}-{space_name}.hf.space/"
42
+ SPACE_URL = "https://huggingface.co/spaces/{space_id}"
43
+ _BOLD_ORANGE = "\033[1m\033[38;5;208m"
44
+ _RESET = "\033[0m"
45
+
46
+
47
+ def raise_if_space_is_frozen_for_logging(space_id: str) -> None:
48
+ try:
49
+ info = huggingface_hub.HfApi().space_info(space_id)
50
+ except RepositoryNotFoundError:
51
+ return
52
+ if getattr(info, "sdk", None) == "static":
53
+ raise RuntimeError(
54
+ f"Cannot log to Hugging Face Space '{space_id}' because it has been frozen "
55
+ f"(it uses the static SDK: a read-only dashboard with no live Trackio server).\n\n"
56
+ f"Use a different space_id for training, or create a new Gradio Trackio Space. "
57
+ f"Freezing converts a live Gradio Space to static after a run; a frozen Space "
58
+ f'cannot accept new logs. See trackio.sync(..., sdk="static") in the Trackio docs.'
59
+ )
60
+
61
+
62
+ def _readme_linked_hub_yaml(dataset_id: str | None) -> str:
63
+ if dataset_id is not None:
64
+ return f"datasets:\n - {dataset_id}\n"
65
+ return ""
66
+
67
+
68
+ _SPACE_APP_PY = "import trackio\ntrackio.show()\n"
69
+
70
+
71
+ def _retry_hf_write(op_name: str, fn, retries: int = 4, initial_delay: float = 1.5):
72
+ delay = initial_delay
73
+ for attempt in range(1, retries + 1):
74
+ try:
75
+ return fn()
76
+ except ReadTimeout:
77
+ if attempt == retries:
78
+ raise
79
+ print(
80
+ f"* {op_name} timed out (attempt {attempt}/{retries}). Retrying in {delay:.1f}s..."
81
+ )
82
+ time.sleep(delay)
83
+ delay = min(delay * 2, 12)
84
+ except HfHubHTTPError as e:
85
+ status = e.response.status_code if e.response is not None else None
86
+ if status is None or status < 500 or attempt == retries:
87
+ raise
88
+ print(
89
+ f"* {op_name} failed with HTTP {status} (attempt {attempt}/{retries}). Retrying in {delay:.1f}s..."
90
+ )
91
+ time.sleep(delay)
92
+ delay = min(delay * 2, 12)
93
+
94
+
95
+ def _get_space_volumes(space_id: str) -> list[Volume]:
96
+ """
97
+ Return mounted volumes for a Space.
98
+
99
+ `HfApi.get_space_runtime()` does not always populate `volumes`, even when the
100
+ mount exists. Fall back to `space_info().runtime.volumes`, which currently
101
+ carries the volume metadata for running Spaces.
102
+ """
103
+ hf_api = huggingface_hub.HfApi()
104
+ runtime = hf_api.get_space_runtime(space_id)
105
+ if runtime.volumes:
106
+ return list(runtime.volumes)
107
+
108
+ info = hf_api.space_info(space_id)
109
+ if info.runtime and info.runtime.volumes:
110
+ return list(info.runtime.volumes)
111
+
112
+ return []
113
+
114
+
115
+ def _get_source_install_dependencies() -> str:
116
+ """Get trackio dependencies from pyproject.toml for source installs."""
117
+ trackio_path = files("trackio")
118
+ pyproject_path = Path(trackio_path).parent / "pyproject.toml"
119
+ with open(pyproject_path, "rb") as f:
120
+ pyproject = tomllib.load(f)
121
+ deps = pyproject["project"]["dependencies"]
122
+ spaces_deps = (
123
+ pyproject["project"].get("optional-dependencies", {}).get("spaces", [])
124
+ )
125
+ mcp_deps = pyproject["project"].get("optional-dependencies", {}).get("mcp", [])
126
+ return "\n".join(deps + spaces_deps + mcp_deps)
127
+
128
+
129
+ def _get_space_install_requirement() -> str:
130
+ return f"trackio[spaces,mcp]=={trackio.__version__}"
131
+
132
+
133
+ def _is_trackio_installed_from_source() -> bool:
134
+ """Check if trackio is installed from source/editable install vs PyPI."""
135
+ try:
136
+ trackio_file = trackio.__file__
137
+ if "site-packages" not in trackio_file and "dist-packages" not in trackio_file:
138
+ return True
139
+
140
+ dist = importlib.metadata.distribution("trackio")
141
+ if dist.files:
142
+ files = list(dist.files)
143
+ has_pth = any(".pth" in str(f) for f in files)
144
+ if has_pth:
145
+ return True
146
+
147
+ return False
148
+ except (
149
+ AttributeError,
150
+ importlib.metadata.PackageNotFoundError,
151
+ importlib.metadata.MetadataError,
152
+ ValueError,
153
+ TypeError,
154
+ ):
155
+ return True
156
+
157
+
158
+ def deploy_as_space(
159
+ space_id: str,
160
+ space_storage: huggingface_hub.SpaceStorage | None = None,
161
+ dataset_id: str | None = None,
162
+ bucket_id: str | None = None,
163
+ private: bool | None = None,
164
+ ):
165
+ if (
166
+ os.getenv("SYSTEM") == "spaces"
167
+ ): # in case a repo with this function is uploaded to spaces
168
+ return
169
+
170
+ if dataset_id is not None and bucket_id is not None:
171
+ raise ValueError(
172
+ "Cannot use bucket volume options together with dataset_id; use one persistence mode."
173
+ )
174
+
175
+ trackio_path = files("trackio")
176
+
177
+ hf_api = huggingface_hub.HfApi()
178
+
179
+ try:
180
+ huggingface_hub.create_repo(
181
+ space_id,
182
+ private=private,
183
+ space_sdk="gradio",
184
+ space_storage=space_storage,
185
+ repo_type="space",
186
+ exist_ok=True,
187
+ )
188
+ except HfHubHTTPError as e:
189
+ if e.response.status_code in [401, 403]: # unauthorized or forbidden
190
+ print("Need 'write' access token to create a Spaces repo.")
191
+ huggingface_hub.login(add_to_git_credential=False)
192
+ huggingface_hub.create_repo(
193
+ space_id,
194
+ private=private,
195
+ space_sdk="gradio",
196
+ space_storage=space_storage,
197
+ repo_type="space",
198
+ exist_ok=True,
199
+ )
200
+ else:
201
+ raise ValueError(f"Failed to create Space: {e}")
202
+
203
+ # We can assume huggingface-hub is available; requirements.txt pins trackio.
204
+ # Make sure necessary dependencies are installed by creating a requirements.txt.
205
+ is_source_install = _is_trackio_installed_from_source()
206
+
207
+ if bucket_id is not None:
208
+ create_bucket_if_not_exists(bucket_id, private=private)
209
+
210
+ with open(Path(trackio_path, "README.md"), "r") as f:
211
+ readme_content = f.read()
212
+ readme_content = readme_content.replace("sdk_version: {GRADIO_VERSION}\n", "")
213
+ readme_content = readme_content.replace("{APP_FILE}", "app.py")
214
+ readme_content = readme_content.replace(
215
+ "{LINKED_HUB_METADATA}", _readme_linked_hub_yaml(dataset_id)
216
+ )
217
+ readme_buffer = io.BytesIO(readme_content.encode("utf-8"))
218
+ hf_api.upload_file(
219
+ path_or_fileobj=readme_buffer,
220
+ path_in_repo="README.md",
221
+ repo_id=space_id,
222
+ repo_type="space",
223
+ )
224
+
225
+ if is_source_install:
226
+ requirements_content = _get_source_install_dependencies()
227
+ else:
228
+ requirements_content = _get_space_install_requirement()
229
+
230
+ requirements_buffer = io.BytesIO(requirements_content.encode("utf-8"))
231
+ hf_api.upload_file(
232
+ path_or_fileobj=requirements_buffer,
233
+ path_in_repo="requirements.txt",
234
+ repo_id=space_id,
235
+ repo_type="space",
236
+ )
237
+
238
+ huggingface_hub.utils.disable_progress_bars()
239
+
240
+ if is_source_install:
241
+ dist_index = (
242
+ Path(trackio.__file__).resolve().parent / "frontend" / "dist" / "index.html"
243
+ )
244
+ if not dist_index.is_file():
245
+ raise ValueError(
246
+ "The Trackio frontend build is missing. From the repository root run "
247
+ "`cd trackio/frontend && npm ci && npm run build`, then deploy again."
248
+ )
249
+ hf_api.upload_folder(
250
+ repo_id=space_id,
251
+ repo_type="space",
252
+ folder_path=trackio_path,
253
+ path_in_repo="trackio",
254
+ ignore_patterns=[
255
+ "README.md",
256
+ "frontend/node_modules/**",
257
+ "frontend/src/**",
258
+ "frontend/.gitignore",
259
+ "frontend/package.json",
260
+ "frontend/package-lock.json",
261
+ "frontend/vite.config.js",
262
+ "frontend/svelte.config.js",
263
+ "**/__pycache__/**",
264
+ "*.pyc",
265
+ ],
266
+ )
267
+
268
+ app_file_content = _SPACE_APP_PY
269
+ app_file_buffer = io.BytesIO(app_file_content.encode("utf-8"))
270
+ hf_api.upload_file(
271
+ path_or_fileobj=app_file_buffer,
272
+ path_in_repo="app.py",
273
+ repo_id=space_id,
274
+ repo_type="space",
275
+ )
276
+
277
+ if hf_token := huggingface_hub.utils.get_token():
278
+ huggingface_hub.add_space_secret(space_id, "HF_TOKEN", hf_token)
279
+ if bucket_id is not None:
280
+ existing = _get_space_volumes(space_id)
281
+ already_mounted = any(
282
+ v.type == "bucket" and v.source == bucket_id and v.mount_path == "/data"
283
+ for v in existing
284
+ )
285
+ if not already_mounted:
286
+ non_bucket = [
287
+ v
288
+ for v in existing
289
+ if not (v.type == "bucket" and v.source == bucket_id)
290
+ ]
291
+ hf_api.set_space_volumes(
292
+ space_id,
293
+ non_bucket
294
+ + [Volume(type="bucket", source=bucket_id, mount_path="/data")],
295
+ )
296
+ print(f"* Attached bucket {bucket_id} at '/data'")
297
+ huggingface_hub.add_space_variable(space_id, "TRACKIO_DIR", "/data/trackio")
298
+ elif dataset_id is not None:
299
+ huggingface_hub.add_space_variable(space_id, "TRACKIO_DATASET_ID", dataset_id)
300
+ if logo_light_url := os.environ.get("TRACKIO_LOGO_LIGHT_URL"):
301
+ huggingface_hub.add_space_variable(
302
+ space_id, "TRACKIO_LOGO_LIGHT_URL", logo_light_url
303
+ )
304
+ if logo_dark_url := os.environ.get("TRACKIO_LOGO_DARK_URL"):
305
+ huggingface_hub.add_space_variable(
306
+ space_id, "TRACKIO_LOGO_DARK_URL", logo_dark_url
307
+ )
308
+ if plot_order := os.environ.get("TRACKIO_PLOT_ORDER"):
309
+ huggingface_hub.add_space_variable(space_id, "TRACKIO_PLOT_ORDER", plot_order)
310
+ if theme := os.environ.get("TRACKIO_THEME"):
311
+ huggingface_hub.add_space_variable(space_id, "TRACKIO_THEME", theme)
312
+ huggingface_hub.add_space_variable(space_id, "GRADIO_MCP_SERVER", "True")
313
+
314
+
315
+ def create_space_if_not_exists(
316
+ space_id: str,
317
+ space_storage: huggingface_hub.SpaceStorage | None = None,
318
+ dataset_id: str | None = None,
319
+ bucket_id: str | None = None,
320
+ private: bool | None = None,
321
+ ) -> None:
322
+ """
323
+ Creates a new Hugging Face Space if it does not exist.
324
+
325
+ Args:
326
+ space_id (`str`):
327
+ The ID of the Space to create.
328
+ space_storage ([`~huggingface_hub.SpaceStorage`], *optional*):
329
+ Choice of persistent storage tier for the Space.
330
+ dataset_id (`str`, *optional*):
331
+ Deprecated. Use `bucket_id` instead.
332
+ bucket_id (`str`, *optional*):
333
+ Full Hub bucket id (`namespace/name`) to attach via the Hub volumes API (platform mount).
334
+ Sets `TRACKIO_DIR` to the mount path.
335
+ private (`bool`, *optional*):
336
+ Whether to make the Space private. If `None` (default), the repo will be
337
+ public unless the organization's default is private. This value is ignored
338
+ if the repo already exists.
339
+ """
340
+ if "/" not in space_id:
341
+ raise ValueError(
342
+ f"Invalid space ID: {space_id}. Must be in the format: username/reponame or orgname/reponame."
343
+ )
344
+ if dataset_id is not None and "/" not in dataset_id:
345
+ raise ValueError(
346
+ f"Invalid dataset ID: {dataset_id}. Must be in the format: username/datasetname or orgname/datasetname."
347
+ )
348
+ if bucket_id is not None and "/" not in bucket_id:
349
+ raise ValueError(
350
+ f"Invalid bucket ID: {bucket_id}. Must be in the format: username/bucketname or orgname/bucketname."
351
+ )
352
+ try:
353
+ huggingface_hub.repo_info(space_id, repo_type="space")
354
+ print(
355
+ f"* Found existing space: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
356
+ )
357
+ return
358
+ except RepositoryNotFoundError:
359
+ pass
360
+ except HfHubHTTPError as e:
361
+ if e.response.status_code in [401, 403]: # unauthorized or forbidden
362
+ print("Need 'write' access token to create a Spaces repo.")
363
+ huggingface_hub.login(add_to_git_credential=False)
364
+ else:
365
+ raise ValueError(f"Failed to create Space: {e}")
366
+
367
+ print(
368
+ f"* Creating new space: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
369
+ )
370
+ deploy_as_space(
371
+ space_id,
372
+ space_storage,
373
+ dataset_id,
374
+ bucket_id,
375
+ private,
376
+ )
377
+ print("* Waiting for Space to be ready...")
378
+ _wait_until_space_running(space_id)
379
+
380
+
381
+ def _wait_until_space_running(space_id: str, timeout: int = 300) -> None:
382
+ hf_api = huggingface_hub.HfApi()
383
+ start = time.time()
384
+ delay = 2
385
+ request_timeout = 45.0
386
+ failure_stages = frozenset(
387
+ ("NO_APP_FILE", "CONFIG_ERROR", "BUILD_ERROR", "RUNTIME_ERROR")
388
+ )
389
+ while time.time() - start < timeout:
390
+ try:
391
+ info = hf_api.space_info(space_id, timeout=request_timeout)
392
+ if info.runtime:
393
+ stage = str(info.runtime.stage)
394
+ if stage in failure_stages:
395
+ raise RuntimeError(
396
+ f"Space {space_id} entered terminal stage {stage}. "
397
+ "Fix README.md or app files; see build logs on the Hub."
398
+ )
399
+ if stage == "RUNNING":
400
+ return
401
+ except RuntimeError:
402
+ raise
403
+ except (huggingface_hub.utils.HfHubHTTPError, httpx.RequestError):
404
+ pass
405
+ time.sleep(delay)
406
+ delay = min(delay * 1.5, 15)
407
+ raise TimeoutError(
408
+ f"Space {space_id} did not reach RUNNING within {timeout}s. "
409
+ "Check status and build logs on the Hub."
410
+ )
411
+
412
+
413
+ def wait_until_space_exists(
414
+ space_id: str,
415
+ ) -> None:
416
+ """
417
+ Blocks the current thread until the Space exists.
418
+
419
+ Args:
420
+ space_id (`str`):
421
+ The ID of the Space to wait for.
422
+
423
+ Raises:
424
+ `TimeoutError`: If waiting for the Space takes longer than expected.
425
+ """
426
+ hf_api = huggingface_hub.HfApi()
427
+ delay = 1
428
+ for _ in range(30):
429
+ try:
430
+ hf_api.space_info(space_id)
431
+ return
432
+ except (huggingface_hub.utils.HfHubHTTPError, httpx.RequestError):
433
+ time.sleep(delay)
434
+ delay = min(delay * 2, 60)
435
+ raise TimeoutError("Waiting for space to exist took longer than expected")
436
+
437
+
438
+ def upload_db_to_space(project: str, space_id: str, force: bool = False) -> None:
439
+ """
440
+ Uploads the database of a local Trackio project to a Hugging Face Space.
441
+
442
+ This uses the Trackio remote client so newer Trackio Spaces can speak the direct
443
+ HTTP API while older Gradio-based Spaces still work through `gradio_client`.
444
+
445
+ Args:
446
+ project (`str`):
447
+ The name of the project to upload.
448
+ space_id (`str`):
449
+ The ID of the Space to upload to.
450
+ force (`bool`, *optional*, defaults to `False`):
451
+ If `True`, overwrites the existing database without prompting. If `False`,
452
+ prompts for confirmation.
453
+ """
454
+ db_path = SQLiteStorage.get_project_db_path(project)
455
+ client = RemoteClient(
456
+ space_id,
457
+ hf_token=huggingface_hub.utils.get_token(),
458
+ httpx_kwargs={"timeout": 90},
459
+ )
460
+
461
+ if not force:
462
+ try:
463
+ existing_projects = client.predict(api_name="/get_all_projects")
464
+ if project in existing_projects:
465
+ response = input(
466
+ f"Database for project '{project}' already exists on Space '{space_id}'. "
467
+ f"Overwrite it? (y/N): "
468
+ )
469
+ if response.lower() not in ["y", "yes"]:
470
+ print("* Upload cancelled.")
471
+ return
472
+ except Exception as e:
473
+ print(f"* Warning: Could not check if project exists on Space: {e}")
474
+ print("* Proceeding with upload...")
475
+
476
+ client.predict(
477
+ api_name="/upload_db_to_space",
478
+ project=project,
479
+ uploaded_db=handle_file(db_path),
480
+ hf_token=huggingface_hub.utils.get_token(),
481
+ )
482
+
483
+
484
+ SYNC_BATCH_SIZE = 500
485
+
486
+
487
+ def sync_incremental(
488
+ project: str,
489
+ space_id: str,
490
+ private: bool | None = None,
491
+ pending_only: bool = False,
492
+ ) -> None:
493
+ """
494
+ Syncs a local Trackio project to a Space via the bulk_log API endpoints
495
+ instead of uploading the entire DB file. Supports incremental sync.
496
+
497
+ Args:
498
+ project: The name of the project to sync.
499
+ space_id: The HF Space ID to sync to.
500
+ private: Whether to make the Space private if creating.
501
+ pending_only: If True, only sync rows tagged with space_id (pending data).
502
+ """
503
+ print(
504
+ f"* Syncing project '{project}' to: {SPACE_URL.format(space_id=space_id)} (please wait...)"
505
+ )
506
+ create_space_if_not_exists(space_id, private=private)
507
+ wait_until_space_exists(space_id)
508
+ hf_token = huggingface_hub.utils.get_token()
509
+ expected_run_counts: Counter[str] = Counter()
510
+
511
+ client = RemoteClient(
512
+ space_id,
513
+ hf_token=hf_token,
514
+ httpx_kwargs={"timeout": 90},
515
+ )
516
+
517
+ if pending_only:
518
+ pending_logs = SQLiteStorage.get_pending_logs(project)
519
+ if pending_logs:
520
+ logs = pending_logs["logs"]
521
+ expected_run_counts.update(log["run"] for log in logs)
522
+ for i in range(0, len(logs), SYNC_BATCH_SIZE):
523
+ batch = logs[i : i + SYNC_BATCH_SIZE]
524
+ print(
525
+ f" Syncing metrics: {min(i + SYNC_BATCH_SIZE, len(logs))}/{len(logs)}..."
526
+ )
527
+ client.predict(api_name="/bulk_log", logs=batch, hf_token=hf_token)
528
+ SQLiteStorage.clear_pending_logs(project, pending_logs["ids"])
529
+
530
+ pending_sys = SQLiteStorage.get_pending_system_logs(project)
531
+ if pending_sys:
532
+ logs = pending_sys["logs"]
533
+ for i in range(0, len(logs), SYNC_BATCH_SIZE):
534
+ batch = logs[i : i + SYNC_BATCH_SIZE]
535
+ print(
536
+ f" Syncing system metrics: {min(i + SYNC_BATCH_SIZE, len(logs))}/{len(logs)}..."
537
+ )
538
+ client.predict(
539
+ api_name="/bulk_log_system", logs=batch, hf_token=hf_token
540
+ )
541
+ SQLiteStorage.clear_pending_system_logs(project, pending_sys["ids"])
542
+
543
+ pending_uploads = SQLiteStorage.get_pending_uploads(project)
544
+ if pending_uploads:
545
+ upload_entries = []
546
+ for u in pending_uploads["uploads"]:
547
+ fp = u["file_path"]
548
+ if os.path.exists(fp):
549
+ upload_entries.append(
550
+ {
551
+ "project": u["project"],
552
+ "run": u["run"],
553
+ "step": u["step"],
554
+ "relative_path": u["relative_path"],
555
+ "uploaded_file": handle_file(fp),
556
+ }
557
+ )
558
+ if upload_entries:
559
+ print(f" Syncing {len(upload_entries)} media files...")
560
+ client.predict(
561
+ api_name="/bulk_upload_media",
562
+ uploads=upload_entries,
563
+ hf_token=hf_token,
564
+ )
565
+ SQLiteStorage.clear_pending_uploads(project, pending_uploads["ids"])
566
+ else:
567
+ all_logs = SQLiteStorage.get_all_logs_for_sync(project)
568
+ if all_logs:
569
+ expected_run_counts.update(log["run"] for log in all_logs)
570
+ for i in range(0, len(all_logs), SYNC_BATCH_SIZE):
571
+ batch = all_logs[i : i + SYNC_BATCH_SIZE]
572
+ print(
573
+ f" Syncing metrics: {min(i + SYNC_BATCH_SIZE, len(all_logs))}/{len(all_logs)}..."
574
+ )
575
+ client.predict(api_name="/bulk_log", logs=batch, hf_token=hf_token)
576
+
577
+ all_sys_logs = SQLiteStorage.get_all_system_logs_for_sync(project)
578
+ if all_sys_logs:
579
+ for i in range(0, len(all_sys_logs), SYNC_BATCH_SIZE):
580
+ batch = all_sys_logs[i : i + SYNC_BATCH_SIZE]
581
+ print(
582
+ f" Syncing system metrics: {min(i + SYNC_BATCH_SIZE, len(all_sys_logs))}/{len(all_sys_logs)}..."
583
+ )
584
+ client.predict(
585
+ api_name="/bulk_log_system", logs=batch, hf_token=hf_token
586
+ )
587
+
588
+ _wait_for_remote_sync(client, project, expected_run_counts)
589
+ SQLiteStorage.set_project_metadata(project, "space_id", space_id)
590
+ print(
591
+ f"* Synced successfully to space: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
592
+ )
593
+
594
+
595
+ def _wait_for_remote_sync(
596
+ client: RemoteClient,
597
+ project: str,
598
+ expected_run_counts: Counter[str],
599
+ timeout: int = 180,
600
+ ) -> None:
601
+ if not expected_run_counts:
602
+ return
603
+
604
+ deadline = time.time() + timeout
605
+ delay = 2
606
+ last_error: Exception | None = None
607
+ pending = dict(expected_run_counts)
608
+
609
+ while time.time() < deadline and pending:
610
+ completed = []
611
+ for run_name, expected_num_logs in pending.items():
612
+ try:
613
+ summary = client.predict(
614
+ project=project, run=run_name, api_name="/get_run_summary"
615
+ )
616
+ if summary.get("num_logs") == expected_num_logs:
617
+ completed.append(run_name)
618
+ except Exception as e:
619
+ last_error = e
620
+ for run_name in completed:
621
+ pending.pop(run_name, None)
622
+ if pending:
623
+ time.sleep(delay)
624
+ delay = min(delay * 1.5, 15)
625
+
626
+ if pending:
627
+ raise TimeoutError(
628
+ f"Remote sync for project '{project}' did not become visible for runs "
629
+ f"{sorted(pending.items())} within {timeout}s. "
630
+ f"Last error: {last_error!r}"
631
+ )
632
+
633
+
634
+ def upload_dataset_for_static(
635
+ project: str,
636
+ dataset_id: str,
637
+ private: bool | None = None,
638
+ ) -> None:
639
+ hf_api = huggingface_hub.HfApi()
640
+
641
+ try:
642
+ huggingface_hub.create_repo(
643
+ dataset_id,
644
+ private=private,
645
+ repo_type="dataset",
646
+ exist_ok=True,
647
+ )
648
+ except HfHubHTTPError as e:
649
+ if e.response.status_code in [401, 403]:
650
+ print("Need 'write' access token to create a Dataset repo.")
651
+ huggingface_hub.login(add_to_git_credential=False)
652
+ huggingface_hub.create_repo(
653
+ dataset_id,
654
+ private=private,
655
+ repo_type="dataset",
656
+ exist_ok=True,
657
+ )
658
+ else:
659
+ raise ValueError(f"Failed to create Dataset: {e}")
660
+
661
+ with tempfile.TemporaryDirectory() as tmp_dir:
662
+ output_dir = Path(tmp_dir)
663
+ SQLiteStorage.export_for_static_space(project, output_dir)
664
+
665
+ media_dir = MEDIA_DIR / project
666
+ if media_dir.exists():
667
+ dest = output_dir / "media"
668
+ shutil.copytree(media_dir, dest)
669
+
670
+ _retry_hf_write(
671
+ "Dataset upload",
672
+ lambda: hf_api.upload_folder(
673
+ repo_id=dataset_id,
674
+ repo_type="dataset",
675
+ folder_path=str(output_dir),
676
+ ),
677
+ )
678
+
679
+ print(f"* Dataset uploaded: https://huggingface.co/datasets/{dataset_id}")
680
+
681
+
682
+ def deploy_as_static_space(
683
+ space_id: str,
684
+ dataset_id: str | None,
685
+ project: str,
686
+ bucket_id: str | None = None,
687
+ private: bool | None = None,
688
+ hf_token: str | None = None,
689
+ ) -> None:
690
+ if os.getenv("SYSTEM") == "spaces":
691
+ return
692
+
693
+ hf_api = huggingface_hub.HfApi()
694
+
695
+ try:
696
+ huggingface_hub.create_repo(
697
+ space_id,
698
+ private=private,
699
+ space_sdk="static",
700
+ repo_type="space",
701
+ exist_ok=True,
702
+ )
703
+ except HfHubHTTPError as e:
704
+ if e.response.status_code in [401, 403]:
705
+ print("Need 'write' access token to create a Spaces repo.")
706
+ huggingface_hub.login(add_to_git_credential=False)
707
+ huggingface_hub.create_repo(
708
+ space_id,
709
+ private=private,
710
+ space_sdk="static",
711
+ repo_type="space",
712
+ exist_ok=True,
713
+ )
714
+ else:
715
+ raise ValueError(f"Failed to create Space: {e}")
716
+
717
+ linked = _readme_linked_hub_yaml(dataset_id)
718
+ readme_content = (
719
+ f"---\nsdk: static\npinned: false\ntags:\n - trackio\n{linked}---\n"
720
+ )
721
+ _retry_hf_write(
722
+ "Static Space README upload",
723
+ lambda: hf_api.upload_file(
724
+ path_or_fileobj=io.BytesIO(readme_content.encode("utf-8")),
725
+ path_in_repo="README.md",
726
+ repo_id=space_id,
727
+ repo_type="space",
728
+ ),
729
+ )
730
+
731
+ trackio_path = files("trackio")
732
+ dist_dir = Path(trackio_path).parent / "trackio" / "frontend" / "dist"
733
+ if not dist_dir.is_dir():
734
+ dist_dir = Path(trackio.__file__).resolve().parent / "frontend" / "dist"
735
+ if not dist_dir.is_dir():
736
+ raise ValueError(
737
+ "The Trackio frontend build is missing. From the repository root run "
738
+ "`cd trackio/frontend && npm ci && npm run build`, then deploy again."
739
+ )
740
+
741
+ _retry_hf_write(
742
+ "Static Space frontend upload",
743
+ lambda: hf_api.upload_folder(
744
+ repo_id=space_id,
745
+ repo_type="space",
746
+ folder_path=str(dist_dir),
747
+ ),
748
+ )
749
+
750
+ config = {
751
+ "mode": "static",
752
+ "project": project,
753
+ "private": bool(private),
754
+ }
755
+ if bucket_id is not None:
756
+ config["bucket_id"] = bucket_id
757
+ if dataset_id is not None:
758
+ config["dataset_id"] = dataset_id
759
+ if hf_token and private:
760
+ config["hf_token"] = hf_token
761
+
762
+ _retry_hf_write(
763
+ "Static Space config upload",
764
+ lambda: hf_api.upload_file(
765
+ path_or_fileobj=io.BytesIO(json_mod.dumps(config).encode("utf-8")),
766
+ path_in_repo="config.json",
767
+ repo_id=space_id,
768
+ repo_type="space",
769
+ ),
770
+ )
771
+
772
+ assets_dir = Path(trackio.__file__).resolve().parent / "assets"
773
+ if assets_dir.is_dir():
774
+ _retry_hf_write(
775
+ "Static Space assets upload",
776
+ lambda: hf_api.upload_folder(
777
+ repo_id=space_id,
778
+ repo_type="space",
779
+ folder_path=str(assets_dir),
780
+ path_in_repo="assets",
781
+ ),
782
+ )
783
+
784
+ print(
785
+ f"* Static Space deployed: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
786
+ )
787
+
788
+
789
+ def sync(
790
+ project: str,
791
+ space_id: str | None = None,
792
+ private: bool | None = None,
793
+ force: bool = False,
794
+ run_in_background: bool = False,
795
+ sdk: str = "gradio",
796
+ dataset_id: str | None = None,
797
+ bucket_id: str | None = None,
798
+ ) -> str:
799
+ """
800
+ Syncs a local Trackio project's database to a Hugging Face Space.
801
+ If the Space does not exist, it will be created. Local data is never deleted.
802
+
803
+ **Freezing:** Passing ``sdk="static"`` deploys a static Space backed by an HF Bucket
804
+ (read-only dashboard, no Gradio server). You can sync the same project again later to
805
+ refresh that static Space. If you want a one-time snapshot of an existing Gradio Space,
806
+ use ``freeze()`` instead.
807
+
808
+ Args:
809
+ project (`str`): The name of the project to upload.
810
+ space_id (`str`, *optional*): The ID of the Space to upload to (e.g., `"username/space_id"`).
811
+ If not provided, checks project metadata first, then generates a random space_id.
812
+ private (`bool`, *optional*):
813
+ Whether to make the Space private. If None (default), the repo will be
814
+ public unless the organization's default is private. This value is ignored
815
+ if the repo already exists.
816
+ force (`bool`, *optional*, defaults to `False`):
817
+ If `True`, overwrite the existing database without prompting for confirmation.
818
+ If `False`, prompt the user before overwriting an existing database.
819
+ run_in_background (`bool`, *optional*, defaults to `False`):
820
+ If `True`, the Space creation and database upload will be run in a background thread.
821
+ If `False`, all the steps will be run synchronously.
822
+ sdk (`str`, *optional*, defaults to `"gradio"`):
823
+ The type of Space to deploy. `"gradio"` deploys a Gradio Space with a live
824
+ server. `"static"` freezes the Space: deploys a static Space that reads from an HF Bucket
825
+ (no server needed).
826
+ dataset_id (`str`, *optional*):
827
+ Deprecated. Use `bucket_id` instead.
828
+ bucket_id (`str`, *optional*):
829
+ The ID of the HF Bucket to sync to. By default, a bucket is auto-generated
830
+ from the space_id.
831
+ Returns:
832
+ `str`: The Space ID of the synced project.
833
+ """
834
+ if sdk not in ("gradio", "static"):
835
+ raise ValueError(f"sdk must be 'gradio' or 'static', got '{sdk}'")
836
+ if space_id is None:
837
+ space_id = SQLiteStorage.get_space_id(project)
838
+ if space_id is None:
839
+ space_id = f"{project}-{get_or_create_project_hash(project)}"
840
+ space_id, dataset_id, bucket_id = preprocess_space_and_dataset_ids(
841
+ space_id, dataset_id, bucket_id
842
+ )
843
+
844
+ def _do_sync():
845
+ try:
846
+ info = huggingface_hub.HfApi().space_info(space_id)
847
+ existing_sdk = info.sdk
848
+ if existing_sdk and existing_sdk != sdk:
849
+ raise ValueError(
850
+ f"Space '{space_id}' is a '{existing_sdk}' Space but sdk='{sdk}' was requested. "
851
+ f"The sdk must match the existing Space type."
852
+ )
853
+ except RepositoryNotFoundError:
854
+ pass
855
+
856
+ if sdk == "static":
857
+ if dataset_id is not None:
858
+ upload_dataset_for_static(project, dataset_id, private=private)
859
+ hf_token = huggingface_hub.utils.get_token() if private else None
860
+ deploy_as_static_space(
861
+ space_id,
862
+ dataset_id,
863
+ project,
864
+ private=private,
865
+ hf_token=hf_token,
866
+ )
867
+ elif bucket_id is not None:
868
+ create_bucket_if_not_exists(bucket_id, private=private)
869
+ upload_project_to_bucket_for_static(project, bucket_id)
870
+ print(
871
+ f"* Project data uploaded to bucket: https://huggingface.co/buckets/{bucket_id}"
872
+ )
873
+ deploy_as_static_space(
874
+ space_id,
875
+ None,
876
+ project,
877
+ bucket_id=bucket_id,
878
+ private=private,
879
+ hf_token=huggingface_hub.utils.get_token() if private else None,
880
+ )
881
+ else:
882
+ if bucket_id is not None:
883
+ create_bucket_if_not_exists(bucket_id, private=private)
884
+ upload_project_to_bucket(project, bucket_id)
885
+ print(
886
+ f"* Project data uploaded to bucket: https://huggingface.co/buckets/{bucket_id}"
887
+ )
888
+ create_space_if_not_exists(
889
+ space_id, bucket_id=bucket_id, private=private
890
+ )
891
+ _wait_for_remote_sync(
892
+ RemoteClient(space_id, verbose=False, httpx_kwargs={"timeout": 90}),
893
+ project,
894
+ Counter(
895
+ log["run"]
896
+ for log in SQLiteStorage.get_all_logs_for_sync(project)
897
+ ),
898
+ )
899
+ else:
900
+ sync_incremental(project, space_id, private=private, pending_only=False)
901
+ SQLiteStorage.set_project_metadata(project, "space_id", space_id)
902
+
903
+ if run_in_background:
904
+ threading.Thread(target=_do_sync).start()
905
+ else:
906
+ _do_sync()
907
+ return space_id
908
+
909
+
910
+ def _get_source_bucket(space_id: str) -> str:
911
+ volumes = _get_space_volumes(space_id)
912
+ for v in volumes:
913
+ if v.type == "bucket" and v.mount_path == "/data":
914
+ return v.source
915
+ raise ValueError(
916
+ f"Space '{space_id}' has no bucket mounted at '/data'. "
917
+ f"freeze() requires the source Space to use bucket storage."
918
+ )
919
+
920
+
921
+ def freeze(
922
+ space_id: str,
923
+ project: str,
924
+ new_space_id: str | None = None,
925
+ private: bool | None = None,
926
+ bucket_id: str | None = None,
927
+ ) -> str:
928
+ """
929
+ Creates a new static Hugging Face Space containing a read-only snapshot of
930
+ the data for the specified project from the source Gradio Space. The data is
931
+ read from the bucket attached to the source Space at freeze time. The original
932
+ Space is not modified, and the new static Space does not automatically reflect
933
+ metrics uploaded to the original Gradio Space after the freeze completes.
934
+
935
+ Args:
936
+ space_id (`str`):
937
+ The ID of the source Gradio Space (e.g., `"username/my-space"` or a
938
+ short repo name with the logged-in namespace inferred, like `init()`).
939
+ Must be a Gradio Space with a bucket mounted at `/data`.
940
+ project (`str`):
941
+ The name of the project whose data to include in the frozen Space.
942
+ new_space_id (`str`, *optional*):
943
+ The ID for the new static Space. If not provided, defaults to
944
+ `"{space_id}_static"`.
945
+ private (`bool`, *optional*):
946
+ Whether to make the new Space private. If None (default), the repo
947
+ will be public unless the organization's default is private.
948
+ bucket_id (`str`, *optional*):
949
+ The ID of the HF Bucket for the new static Space's data storage.
950
+ If not provided, one is auto-generated from the new Space ID.
951
+
952
+ Returns:
953
+ `str`: The Space ID of the newly created static Space.
954
+ """
955
+ space_id, _, _ = preprocess_space_and_dataset_ids(space_id, None, None)
956
+
957
+ try:
958
+ info = huggingface_hub.HfApi().space_info(space_id)
959
+ if info.sdk != "gradio":
960
+ raise ValueError(
961
+ f"Space '{space_id}' is not a Gradio Space (sdk='{info.sdk}'). "
962
+ f"freeze() requires a Gradio Space as the source."
963
+ )
964
+ except RepositoryNotFoundError:
965
+ raise ValueError(
966
+ f"Space '{space_id}' not found. Provide an existing Gradio Space ID."
967
+ )
968
+
969
+ source_bucket_id = _get_source_bucket(space_id)
970
+ print(f"* Reading project data from bucket: {source_bucket_id}")
971
+
972
+ if new_space_id is None:
973
+ new_space_id = f"{space_id}_static"
974
+ new_space_id, _dataset_id, bucket_id = preprocess_space_and_dataset_ids(
975
+ new_space_id, None, bucket_id
976
+ )
977
+
978
+ hf_api = huggingface_hub.HfApi()
979
+ try:
980
+ dest_info = hf_api.space_info(new_space_id)
981
+ tags = dest_info.tags or []
982
+ if dest_info.sdk != "static" or "trackio" not in tags:
983
+ raise ValueError(
984
+ f"Space '{new_space_id}' already exists and is not a Trackio static Space "
985
+ f"(sdk='{dest_info.sdk}', tags={tags}). Choose a different new_space_id "
986
+ f"or delete the existing Space first."
987
+ )
988
+ except RepositoryNotFoundError:
989
+ pass
990
+
991
+ create_bucket_if_not_exists(bucket_id, private=private)
992
+ export_from_bucket_for_static(source_bucket_id, bucket_id, project)
993
+ print(
994
+ f"* Project data uploaded to bucket: https://huggingface.co/buckets/{bucket_id}"
995
+ )
996
+ deploy_as_static_space(
997
+ new_space_id,
998
+ None,
999
+ project,
1000
+ bucket_id=bucket_id,
1001
+ private=private,
1002
+ hf_token=huggingface_hub.utils.get_token() if private else None,
1003
+ )
1004
+ return new_space_id
trackio/dummy_commit_scheduler.py ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from concurrent.futures import Future
2
+
3
+
4
+ class DummyCommitSchedulerLock:
5
+ def __enter__(self):
6
+ return None
7
+
8
+ def __exit__(self, exception_type, exception_value, exception_traceback):
9
+ pass
10
+
11
+
12
+ class DummyCommitScheduler:
13
+ def __init__(self):
14
+ self.lock = DummyCommitSchedulerLock()
15
+
16
+ def trigger(self) -> Future:
17
+ fut: Future = Future()
18
+ fut.set_result(None)
19
+ return fut
trackio/exceptions.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ class TrackioAPIError(Exception):
2
+ pass
trackio/frontend/dist/assets/index-CicmrdCP.css ADDED
@@ -0,0 +1 @@
 
 
1
+ :root{--primary-50: #fff7ed;--primary-100: #ffedd5;--primary-200: #fed7aa;--primary-300: #fdba74;--primary-400: #fb923c;--primary-500: #f97316;--primary-600: #ea580c;--primary-700: #c2410c;--primary-800: #9a3412;--primary-900: #7c2d12;--primary-950: #6c2e12;--secondary-50: #eff6ff;--secondary-100: #dbeafe;--secondary-200: #bfdbfe;--secondary-300: #93c5fd;--secondary-400: #60a5fa;--secondary-500: #3b82f6;--secondary-600: #2563eb;--secondary-700: #1d4ed8;--secondary-800: #1e40af;--secondary-900: #1e3a8a;--secondary-950: #1d3660;--neutral-50: #f9fafb;--neutral-100: #f3f4f6;--neutral-200: #e5e7eb;--neutral-300: #d1d5db;--neutral-400: #9ca3af;--neutral-500: #6b7280;--neutral-600: #4b5563;--neutral-700: #374151;--neutral-800: #1f2937;--neutral-900: #111827;--neutral-950: #0b0f19;--size-0-5: 2px;--size-1: 4px;--size-2: 8px;--size-3: 12px;--size-4: 16px;--size-5: 20px;--size-6: 24px;--size-8: 32px;--size-14: 56px;--size-16: 64px;--size-28: 112px;--size-full: 100%;--spacing-xxs: 1px;--spacing-xs: 2px;--spacing-sm: 4px;--spacing-md: 6px;--spacing-lg: 8px;--spacing-xl: 10px;--spacing-xxl: 16px;--radius-xxs: 1px;--radius-xs: 2px;--radius-sm: 3px;--radius-md: 4px;--radius-lg: 5px;--radius-xl: 8px;--radius-xxl: 12px;--text-xxs: 9px;--text-xs: 10px;--text-sm: 12px;--text-md: 14px;--text-lg: 16px;--text-xl: 22px;--text-xxl: 26px;--line-sm: 1.4;--background-fill-primary: white;--background-fill-secondary: var(--neutral-50);--body-text-color: var(--neutral-900);--body-text-color-subdued: var(--neutral-600);--border-color-primary: var(--neutral-200);--color-accent: var(--primary-500);--color-accent-soft: var(--primary-50);--shadow-drop: rgba(0, 0, 0, .05) 0px 1px 2px 0px;--shadow-drop-lg: 0 1px 3px 0 rgb(0 0 0 / .1), 0 1px 2px -1px rgb(0 0 0 / .1);--shadow-inset: rgba(0, 0, 0, .05) 0px 2px 4px 0px inset;--shadow-spread: 3px;--block-title-text-color: var(--neutral-500);--block-title-text-size: var(--text-md);--block-title-text-weight: 400;--block-info-text-color: var(--body-text-color-subdued);--block-info-text-size: var(--text-sm);--input-background-fill: white;--input-background-fill-focus: var(--primary-500);--input-border-color: var(--border-color-primary);--input-border-color-focus: var(--primary-300);--input-border-width: 1px;--input-padding: var(--spacing-xl);--input-placeholder-color: var(--neutral-400);--input-radius: var(--radius-lg);--input-shadow: 0 0 0 var(--shadow-spread) transparent, var(--shadow-inset);--input-shadow-focus: 0 0 0 var(--shadow-spread) var(--primary-50), var(--shadow-inset);--input-text-size: var(--text-md);--checkbox-background-color: var(--background-fill-primary);--checkbox-background-color-focus: var(--checkbox-background-color);--checkbox-background-color-hover: var(--checkbox-background-color);--checkbox-background-color-selected: var(--primary-600);--checkbox-border-color: var(--neutral-300);--checkbox-border-color-focus: var(--primary-500);--checkbox-border-color-hover: var(--neutral-300);--checkbox-border-color-selected: var(--primary-600);--checkbox-border-radius: var(--radius-sm);--checkbox-border-width: var(--input-border-width);--checkbox-label-gap: var(--spacing-lg);--checkbox-label-padding: var(--spacing-md) calc(2 * var(--spacing-md));--checkbox-label-text-size: var(--text-md);--checkbox-shadow: var(--input-shadow);--checkbox-check: url("data:image/svg+xml,%3csvg viewBox='0 0 16 16' fill='white' xmlns='http://www.w3.org/2000/svg'%3e%3cpath d='M12.207 4.793a1 1 0 010 1.414l-5 5a1 1 0 01-1.414 0l-2-2a1 1 0 011.414-1.414L6.5 9.086l4.293-4.293a1 1 0 011.414 0z'/%3e%3c/svg%3e");--slider-color: var(--primary-500);--container-radius: var(--radius-lg);--layer-top: 9999}.navbar.svelte-d8j1hi{display:flex;align-items:stretch;border-bottom:1px solid var(--border-color-primary, #e5e7eb);background:var(--background-fill-primary, white);padding:0;flex-shrink:0;min-height:44px}.nav-spacer.svelte-d8j1hi{flex:1 1 0;min-width:0}.nav-tabs.svelte-d8j1hi{display:flex;gap:0;flex-shrink:0;padding-right:8px}.nav-link.svelte-d8j1hi{padding:10px 16px;border:none;background:none;color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-md, 14px);cursor:pointer;white-space:nowrap;border-bottom:2px solid transparent;transition:color .15s;font-weight:400}.nav-link.svelte-d8j1hi:hover{color:var(--body-text-color, #1f2937)}.nav-link.active.svelte-d8j1hi{color:var(--body-text-color, #1f2937);border-bottom-color:var(--body-text-color, #1f2937);font-weight:500}.settings-btn.svelte-d8j1hi{display:flex;align-items:center;gap:6px}.settings-btn.svelte-d8j1hi svg:where(.svelte-d8j1hi){flex-shrink:0}.checkbox-group.svelte-17gmtkf{display:flex;flex-direction:column}.checkbox-item.svelte-17gmtkf{display:flex;align-items:center;gap:8px;padding:3px 0;cursor:pointer;font-size:13px}.checkbox-item.svelte-17gmtkf input[type=checkbox]:where(.svelte-17gmtkf){-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;margin:0;border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);box-shadow:var(--checkbox-shadow);cursor:pointer;flex-shrink:0;transition:background-color .15s,border-color .15s}.checkbox-item.svelte-17gmtkf input[type=checkbox]:where(.svelte-17gmtkf):checked{background-image:var(--checkbox-check);background-color:var(--checkbox-background-color-selected, #f97316);border-color:var(--checkbox-border-color-selected, #f97316)}.checkbox-item.svelte-17gmtkf input[type=checkbox]:where(.svelte-17gmtkf):hover{border-color:var(--checkbox-border-color-hover, #d1d5db)}.color-dot.svelte-17gmtkf{width:10px;height:10px;border-radius:50%;flex-shrink:0}.run-name.svelte-17gmtkf{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:var(--body-text-color, #1f2937)}.dropdown-container.svelte-kgylqb{width:100%}.label.svelte-kgylqb{display:block;font-size:13px;font-weight:500;color:var(--body-text-color-subdued, #6b7280);margin-bottom:6px}.info.svelte-kgylqb{display:block;font-size:12px;color:var(--body-text-color-subdued, #9ca3af);margin-bottom:4px}.wrap.svelte-kgylqb{position:relative;border-radius:var(--input-radius, 8px);background:var(--input-background-fill, white);border:1px solid var(--border-color-primary, #e5e7eb);transition:border-color .15s,box-shadow .15s}.wrap.focused.svelte-kgylqb{border-color:var(--input-border-color-focus, #fdba74);box-shadow:0 0 0 2px var(--primary-50, #fff7ed)}.wrap-inner.svelte-kgylqb{display:flex;position:relative;align-items:center;padding:0 10px}.secondary-wrap.svelte-kgylqb{display:flex;flex:1;align-items:center}input.svelte-kgylqb{margin:0;outline:none;border:none;background:inherit;width:100%;color:var(--body-text-color, #1f2937);font-size:13px;font-family:inherit;padding:7px 0}input.svelte-kgylqb::placeholder{color:var(--input-placeholder-color, #9ca3af)}input[readonly].svelte-kgylqb{cursor:pointer}.icon-wrap.svelte-kgylqb{color:var(--body-text-color-subdued, #9ca3af);width:16px;flex-shrink:0;pointer-events:none}.options.svelte-kgylqb{position:fixed;z-index:var(--layer-top, 9999);margin:0;padding:4px 0;box-shadow:0 4px 12px #0000001f;border-radius:var(--input-radius, 8px);border:1px solid var(--border-color-primary, #e5e7eb);background:var(--background-fill-primary, white);min-width:fit-content;overflow:auto;color:var(--body-text-color, #1f2937);list-style:none}.item.svelte-kgylqb{display:flex;cursor:pointer;padding:6px 10px;font-size:13px;word-break:break-word}.item.svelte-kgylqb:hover,.item.active.svelte-kgylqb{background:var(--background-fill-secondary, #f9fafb)}.item.selected.svelte-kgylqb{font-weight:500}.check-mark.svelte-kgylqb{padding-right:6px;min-width:16px;font-size:12px}.check-mark.hide.svelte-kgylqb{visibility:hidden}.checkbox-container.svelte-oj84db{display:flex;align-items:center;gap:8px;cursor:pointer;margin:8px 0}.label-text.svelte-oj84db{color:var(--body-text-color, #1f2937);font-size:13px;line-height:1.4}input[type=checkbox].svelte-oj84db{--ring-color: transparent;position:relative;-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;box-shadow:var(--checkbox-shadow);border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);flex-shrink:0;cursor:pointer;transition:background-color .15s,border-color .15s}input[type=checkbox].svelte-oj84db:checked,input[type=checkbox].svelte-oj84db:checked:hover,input[type=checkbox].svelte-oj84db:checked:focus{background-image:var(--checkbox-check);background-color:var(--checkbox-background-color-selected, #f97316);border-color:var(--checkbox-border-color-selected, #f97316)}input[type=checkbox].svelte-oj84db:hover{border-color:var(--checkbox-border-color-hover, #d1d5db);background-color:var(--checkbox-background-color-hover, white)}input[type=checkbox].svelte-oj84db:focus{border-color:var(--checkbox-border-color-focus, #f97316);background-color:var(--checkbox-background-color-focus, white);outline:none}.slider-wrap.svelte-wei6ev{display:flex;flex-direction:column;width:100%}.head.svelte-wei6ev{margin-bottom:4px;display:flex;justify-content:space-between;align-items:center;width:100%}.label.svelte-wei6ev{flex:1;font-size:13px;font-weight:500;color:var(--body-text-color-subdued, #6b7280)}.info.svelte-wei6ev{display:block;font-size:12px;color:var(--body-text-color-subdued, #9ca3af);margin-bottom:4px}.slider-input-container.svelte-wei6ev{display:flex;align-items:center;gap:6px}input[type=range].svelte-wei6ev{-webkit-appearance:none;-moz-appearance:none;appearance:none;width:100%;cursor:pointer;outline:none;border-radius:var(--radius-xl, 12px);min-width:var(--size-28, 112px);background:transparent}input[type=range].svelte-wei6ev::-webkit-slider-runnable-track{height:6px;border-radius:var(--radius-xl, 12px);background:linear-gradient(to right,var(--slider-color, #f97316) var(--range_progress, 50%),var(--neutral-200, #e5e7eb) var(--range_progress, 50%))}input[type=range].svelte-wei6ev::-webkit-slider-thumb{-webkit-appearance:none;-moz-appearance:none;appearance:none;height:16px;width:16px;background-color:var(--slider-color, #f97316);border:2px solid var(--background-fill-primary, white);border-radius:50%;margin-top:-5px;box-shadow:0 0 0 1px var(--border-color-primary, rgba(0, 0, 0, .08)),0 1px 3px #0003}input[type=range].svelte-wei6ev::-moz-range-track{height:6px;background:var(--neutral-200, #e5e7eb);border-radius:var(--radius-xl, 12px)}input[type=range].svelte-wei6ev::-moz-range-thumb{-webkit-appearance:none;-moz-appearance:none;appearance:none;height:16px;width:16px;background-color:var(--slider-color, #f97316);border:2px solid var(--background-fill-primary, white);border-radius:50%;box-shadow:0 0 0 1px var(--border-color-primary, rgba(0, 0, 0, .08)),0 1px 3px #0003}input[type=range].svelte-wei6ev::-moz-range-progress{height:6px;background-color:var(--slider-color, #f97316);border-radius:var(--radius-xl, 12px)}.bound.svelte-wei6ev{font-size:11px;color:var(--body-text-color-subdued, #9ca3af);min-width:12px;text-align:center}.textbox-container.svelte-6yncpg{width:100%}.label.svelte-6yncpg{display:block;font-size:13px;font-weight:500;color:var(--body-text-color-subdued, #6b7280);margin-bottom:6px}.info.svelte-6yncpg{display:block;font-size:12px;color:var(--body-text-color-subdued, #9ca3af);margin-bottom:4px}.input-wrap.svelte-6yncpg{border-radius:var(--input-radius, 8px);background:var(--input-background-fill, white);border:1px solid var(--border-color-primary, #e5e7eb);transition:border-color .15s,box-shadow .15s}.input-wrap.svelte-6yncpg:focus-within{border-color:var(--input-border-color-focus, #fdba74);box-shadow:0 0 0 2px var(--primary-50, #fff7ed)}input.svelte-6yncpg{width:100%;padding:7px 10px;outline:none;border:none;background:transparent;color:var(--body-text-color, #1f2937);font-size:13px;font-family:inherit;border-radius:var(--input-radius, 8px)}input.svelte-6yncpg::placeholder{color:var(--input-placeholder-color, #9ca3af)}.sidebar.svelte-181dlmc{width:290px;min-width:290px;background:var(--background-fill-primary, white);border-right:1px solid var(--border-color-primary, #e5e7eb);display:flex;flex-direction:column;position:relative;overflow:hidden;transition:width .2s,min-width .2s}.sidebar.collapsed.svelte-181dlmc{width:40px;min-width:40px}.toggle-btn.svelte-181dlmc{position:absolute;top:12px;right:8px;z-index:10;border:none;background:none;color:var(--body-text-color-subdued, #9ca3af);cursor:pointer;padding:4px;display:flex;align-items:center;justify-content:center;border-radius:var(--radius-sm, 4px);transition:color .15s,background-color .15s}.toggle-btn.svelte-181dlmc:hover{color:var(--body-text-color, #1f2937);background-color:var(--background-fill-secondary, #f9fafb)}.sidebar-content.svelte-181dlmc{padding:16px;flex:1;min-height:0;display:flex;flex-direction:column}.sidebar-scroll.svelte-181dlmc{overflow-y:auto;flex:1;min-height:0}.oauth-footer.svelte-181dlmc{flex-shrink:0;margin-top:12px;padding-top:12px;border-top:1px solid var(--border-color-primary, #e5e7eb)}.readonly-footer.svelte-181dlmc{flex-shrink:0;margin-top:12px;padding-top:12px;border-top:1px solid var(--border-color-primary, #e5e7eb);display:flex;align-items:center;gap:8px;flex-wrap:wrap}.readonly-badge.svelte-181dlmc{display:inline-flex;align-items:center;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:999px;padding:2px 8px;font-size:10px;letter-spacing:.06em;font-weight:600;color:var(--body-text-color-subdued, #6b7280);background:var(--background-fill-secondary, #f9fafb)}.readonly-link.svelte-181dlmc{font-size:12px;color:var(--body-text-color-subdued, #6b7280);text-decoration:none;max-width:100%;overflow-wrap:anywhere}.readonly-link.svelte-181dlmc:hover{color:var(--body-text-color, #1f2937);text-decoration:underline}.oauth-line.svelte-181dlmc{margin:0;font-size:12px;line-height:1.4;color:var(--body-text-color-subdued, #6b7280)}.oauth-warn.svelte-181dlmc{color:var(--body-text-color, #92400e)}.hf-login-btn.svelte-181dlmc{display:inline-flex;align-items:center;justify-content:center;gap:8px;width:100%;padding:8px 12px;font-size:13px;font-weight:600;color:#fff;background:#141c2e;border-radius:var(--radius-lg, 8px);text-decoration:none;border:none;cursor:pointer;box-sizing:border-box}.hf-login-btn.svelte-181dlmc:hover{background:#283042}.hf-logo.svelte-181dlmc{width:20px;height:20px;flex-shrink:0}.oauth-hint.svelte-181dlmc{margin:8px 0 0;font-size:11px;line-height:1.35;color:var(--body-text-color-subdued, #9ca3af)}.oauth-signed-in.svelte-181dlmc{margin:0;font-size:12px;color:var(--body-text-color-subdued, #6b7280)}.oauth-logout.svelte-181dlmc{font-size:12px;color:var(--body-text-color-subdued, #9ca3af);text-decoration:none;cursor:pointer}.oauth-logout.svelte-181dlmc:hover{text-decoration:underline;color:var(--body-text-color, #1f2937)}.logo-section.svelte-181dlmc{margin-bottom:20px}.logo.svelte-181dlmc{width:80%;max-width:200px}.section.svelte-181dlmc{margin-bottom:18px}.section-label.svelte-181dlmc{font-size:13px;font-weight:500;color:var(--body-text-color-subdued, #6b7280)}.locked-project.svelte-181dlmc{margin-top:4px;font-size:13px;font-weight:500;color:var(--body-text-color, #1f2937);padding:8px 10px;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-md, 6px);background:var(--background-fill-secondary, #f9fafb)}.runs-header.svelte-181dlmc{display:flex;align-items:center;justify-content:space-between;margin-bottom:6px}.select-all-label.svelte-181dlmc{display:flex;align-items:center;gap:6px;cursor:pointer}.select-all-cb.svelte-181dlmc{-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);cursor:pointer;flex-shrink:0;position:relative;transition:background-color .15s,border-color .15s}.select-all-cb.svelte-181dlmc:checked{background-color:var(--checkbox-background-color-selected, var(--color-accent, #f97316));border-color:var(--checkbox-background-color-selected, var(--color-accent, #f97316));background-image:var(--checkbox-check)}.select-all-cb.svelte-181dlmc:indeterminate{background-color:var(--checkbox-background-color-selected, var(--color-accent, #f97316));border-color:var(--checkbox-background-color-selected, var(--color-accent, #f97316));background-image:url("data:image/svg+xml,%3Csvg viewBox='0 0 16 16' fill='white' xmlns='http://www.w3.org/2000/svg'%3E%3Crect x='3' y='7' width='10' height='2' rx='1'/%3E%3C/svg%3E");background-size:12px;background-position:center;background-repeat:no-repeat}.latest-toggle.svelte-181dlmc{display:flex;align-items:center;gap:6px;font-size:12px;color:var(--body-text-color-subdued, #6b7280);cursor:pointer}.latest-toggle.svelte-181dlmc input[type=checkbox]:where(.svelte-181dlmc){-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;margin:0;border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);box-shadow:var(--checkbox-shadow);cursor:pointer;flex-shrink:0;transition:background-color .15s,border-color .15s}.latest-toggle.svelte-181dlmc input[type=checkbox]:where(.svelte-181dlmc):checked{background-image:var(--checkbox-check);background-color:var(--checkbox-background-color-selected, #f97316);border-color:var(--checkbox-border-color-selected, #f97316)}.checkbox-list.svelte-181dlmc{max-height:300px;overflow-y:auto;margin-top:8px}.alert-panel.svelte-x5aqew{position:fixed;bottom:16px;right:16px;width:380px;max-height:400px;background:var(--background-fill-primary, white);border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);box-shadow:var(--shadow-drop-lg);z-index:1000;overflow:hidden;display:flex;flex-direction:column}.alert-panel.collapsed.svelte-x5aqew{max-height:none}.alert-header.svelte-x5aqew{padding:10px 12px;border:none;border-bottom:1px solid var(--border-color-primary, #e5e7eb);background:none;width:100%;display:flex;align-items:center;justify-content:space-between;cursor:pointer;gap:8px}.alert-panel.collapsed.svelte-x5aqew .alert-header:where(.svelte-x5aqew){border-bottom:none}.collapse-icon.svelte-x5aqew{color:var(--body-text-color-subdued, #9ca3af);flex-shrink:0;transition:transform .15s}.collapse-icon.rotated.svelte-x5aqew{transform:rotate(-90deg)}.alert-title.svelte-x5aqew{font-size:13px;font-weight:600;color:var(--body-text-color, #1f2937)}.filter-pills.svelte-x5aqew{display:flex;gap:4px}.pill.svelte-x5aqew{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-xxl, 22px);padding:2px 8px;font-size:11px;background:var(--background-fill-secondary, #f9fafb);color:var(--body-text-color-subdued, #6b7280);cursor:pointer}.pill.active.svelte-x5aqew{background:var(--color-accent, #f97316);color:#fff;border-color:var(--color-accent, #f97316)}.alert-list.svelte-x5aqew{overflow-y:auto;flex:1}.alert-item.svelte-x5aqew{border-bottom:1px solid var(--neutral-100, #f3f4f6)}.alert-row.svelte-x5aqew{display:flex;align-items:center;gap:8px;width:100%;padding:8px 12px;border:none;background:none;text-align:left;cursor:pointer;font-size:var(--text-sm, 12px)}.alert-row.svelte-x5aqew:hover{background:var(--background-fill-secondary, #f9fafb)}.alert-text.svelte-x5aqew{flex:1;color:var(--body-text-color, #1f2937)}.alert-meta.svelte-x5aqew{font-size:var(--text-xs, 10px);color:var(--body-text-color-subdued, #9ca3af);white-space:nowrap}.alert-detail.svelte-x5aqew{padding:4px 12px 8px 32px;font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #6b7280)}.plot-container.svelte-9thu1j{min-width:350px;flex:1;background:var(--background-fill-primary, white);border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);padding:12px;overflow:hidden;position:relative}.plot-container[draggable=true].svelte-9thu1j{cursor:grab}.plot-container[draggable=true].svelte-9thu1j:active{cursor:grabbing}.hidden-plot.svelte-9thu1j{visibility:hidden;height:0;padding:0;margin:0;border:none;overflow:hidden;pointer-events:none}.drag-handle.svelte-9thu1j{position:absolute;top:8px;left:8px;color:var(--body-text-color-subdued, #9ca3af);opacity:0;transition:opacity .15s;z-index:5}.plot-container.svelte-9thu1j:hover .drag-handle:where(.svelte-9thu1j){opacity:.5}.drag-handle.svelte-9thu1j:hover{opacity:1!important}.plot-toolbar.svelte-9thu1j{position:absolute;top:8px;right:8px;display:flex;gap:4px;z-index:5;opacity:0;transition:opacity .15s}.plot-container.svelte-9thu1j:hover .plot-toolbar:where(.svelte-9thu1j){opacity:1}.toolbar-btn.svelte-9thu1j{border:1px solid var(--border-color-primary, #e5e7eb);background:var(--background-fill-primary, white);color:var(--body-text-color-subdued, #6b7280);cursor:pointer;padding:4px 6px;border-radius:var(--radius-sm, 4px);display:flex;align-items:center;justify-content:center}.toolbar-btn.svelte-9thu1j:hover{background:var(--neutral-100, #f3f4f6);color:var(--body-text-color, #1f2937)}.plot-chart-wrap.svelte-9thu1j{position:relative;width:100%}.plot-chart-wrap--fs.svelte-9thu1j{flex:1;min-height:0;display:flex;flex-direction:column}.reset-zoom-btn.svelte-9thu1j{position:absolute;bottom:1px;right:1px;z-index:6;display:inline-flex;align-items:center;justify-content:center;margin:0;min-width:52px;padding:5px 12px 5px 10px;border:none;border-radius:4px;background:transparent;color:var(--body-text-color-subdued, #334155);cursor:pointer;opacity:.92;transform:translateY(6px);transition:opacity .15s ease,color .15s ease,background .15s ease;box-shadow:none}.reset-zoom-btn.svelte-9thu1j:hover{opacity:1;color:var(--body-text-color, #0f172a);background:var(--background-fill-secondary, rgba(226, 232, 240, .85));transform:translateY(6px)}.reset-zoom-btn.svelte-9thu1j svg:where(.svelte-9thu1j){display:block;flex-shrink:0;filter:drop-shadow(0 0 .5px rgba(255,255,255,.95))}.plot.svelte-9thu1j{width:100%}.plot.svelte-9thu1j .vega-embed{width:100%!important}.plot.svelte-9thu1j .vega-embed summary{display:none}.fullscreen-host.svelte-9thu1j{position:fixed;top:0;right:0;bottom:0;left:0;z-index:10000;box-sizing:border-box;display:flex;flex-direction:column;background:var(--background-fill-primary, white);padding:12px;gap:8px;pointer-events:auto}.fullscreen-host.svelte-9thu1j:fullscreen{width:100%;height:100%}.fullscreen-host.svelte-9thu1j:-webkit-full-screen{width:100%;height:100%}.fullscreen-toolbar.svelte-9thu1j{flex-shrink:0;display:flex;justify-content:flex-end;gap:4px;z-index:5}.fullscreen-chart-wrap.svelte-9thu1j{flex:1;min-height:0;display:flex;flex-direction:column}.fullscreen-legend.svelte-9thu1j{flex-shrink:0}.fullscreen-plot.svelte-9thu1j{flex:1;min-height:0;width:100%;overflow:hidden}.fullscreen-plot.svelte-9thu1j .vega-embed{width:100%!important;height:100%!important;min-height:0;display:flex;flex-direction:column}.fullscreen-plot.svelte-9thu1j .vega-embed .vega-view{flex:1;min-height:0}.fullscreen-plot.svelte-9thu1j .vega-embed summary{display:none}.custom-legend.svelte-9thu1j{display:flex;align-items:center;justify-content:center;gap:12px;padding:6px 0 0;flex-wrap:wrap}.legend-title.svelte-9thu1j{font-size:11px;color:var(--body-text-color-subdued, #6b7280);font-weight:600}.legend-item.svelte-9thu1j{display:flex;align-items:center;gap:4px}.legend-dot.svelte-9thu1j{width:10px;height:10px;border-radius:50%;flex-shrink:0}.legend-label.svelte-9thu1j{font-size:11px;color:var(--body-text-color-subdued, #6b7280)}.plot-container.svelte-1swghqy{min-width:350px;flex:1;background:var(--background-fill-primary, white);border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);padding:12px;overflow:hidden;position:relative}.plot-container[draggable=true].svelte-1swghqy{cursor:grab}.plot-container[draggable=true].svelte-1swghqy:active{cursor:grabbing}.hidden-plot.svelte-1swghqy{visibility:hidden;height:0;padding:0;margin:0;border:none;overflow:hidden;pointer-events:none}.drag-handle.svelte-1swghqy{position:absolute;top:8px;left:8px;color:var(--body-text-color-subdued, #9ca3af);opacity:0;transition:opacity .15s;z-index:5}.plot-container.svelte-1swghqy:hover .drag-handle:where(.svelte-1swghqy){opacity:.5}.drag-handle.svelte-1swghqy:hover{opacity:1!important}.plot-toolbar.svelte-1swghqy{position:absolute;top:8px;right:8px;display:flex;gap:4px;z-index:5;opacity:0;transition:opacity .15s}.plot-container.svelte-1swghqy:hover .plot-toolbar:where(.svelte-1swghqy){opacity:1}.toolbar-btn.svelte-1swghqy{border:1px solid var(--border-color-primary, #e5e7eb);background:var(--background-fill-primary, white);color:var(--body-text-color-subdued, #6b7280);cursor:pointer;padding:4px 6px;border-radius:var(--radius-sm, 4px);display:flex;align-items:center;justify-content:center}.toolbar-btn.svelte-1swghqy:hover{background:var(--neutral-100, #f3f4f6);color:var(--body-text-color, #1f2937)}.plot-chart-wrap.svelte-1swghqy{position:relative;width:100%}.plot-chart-wrap--fs.svelte-1swghqy{flex:1;min-height:0;display:flex;flex-direction:column}.plot.svelte-1swghqy{width:100%}.plot.svelte-1swghqy .vega-embed{width:100%!important}.plot.svelte-1swghqy .vega-embed summary{display:none}.fullscreen-host.svelte-1swghqy{position:fixed;top:0;right:0;bottom:0;left:0;z-index:10000;box-sizing:border-box;display:flex;flex-direction:column;background:var(--background-fill-primary, white);padding:12px;gap:8px;pointer-events:auto}.fullscreen-host.svelte-1swghqy:fullscreen{width:100%;height:100%}.fullscreen-host.svelte-1swghqy:-webkit-full-screen{width:100%;height:100%}.fullscreen-toolbar.svelte-1swghqy{flex-shrink:0;display:flex;justify-content:flex-end;gap:4px;z-index:5}.fullscreen-chart-wrap.svelte-1swghqy{flex:1;min-height:0;display:flex;flex-direction:column}.fullscreen-legend.svelte-1swghqy{flex-shrink:0}.fullscreen-plot.svelte-1swghqy{flex:1;min-height:0;width:100%;overflow:hidden}.fullscreen-plot.svelte-1swghqy .vega-embed{width:100%!important;height:100%!important;min-height:0;display:flex;flex-direction:column}.fullscreen-plot.svelte-1swghqy .vega-embed .vega-view{flex:1;min-height:0}.fullscreen-plot.svelte-1swghqy .vega-embed summary{display:none}.custom-legend.svelte-1swghqy{display:flex;align-items:center;justify-content:center;gap:12px;padding:6px 0 0;flex-wrap:wrap}.legend-title.svelte-1swghqy{font-size:11px;color:var(--body-text-color-subdued, #6b7280);font-weight:600}.legend-item.svelte-1swghqy{display:flex;align-items:center;gap:4px}.legend-dot.svelte-1swghqy{width:10px;height:10px;border-radius:50%;flex-shrink:0}.legend-label.svelte-1swghqy{font-size:11px;color:var(--body-text-color-subdued, #6b7280)}.accordion.svelte-1jep0a{margin-bottom:12px;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);background:var(--background-fill-primary, white);overflow:hidden}.accordion-hidden.svelte-1jep0a{margin-bottom:8px}.accordion-header.svelte-1jep0a{display:flex;align-items:center;gap:8px;width:100%;padding:10px 14px;border:none;background:var(--background-fill-primary, white);color:var(--body-text-color, #1f2937);font-size:var(--text-md, 14px);font-weight:600;cursor:pointer;text-align:left}.accordion-header.svelte-1jep0a:hover{background:var(--background-fill-secondary, #f9fafb)}.arrow.svelte-1jep0a{font-size:14px;transition:transform .15s;color:var(--body-text-color, #1f2937);display:inline-block}.arrow.svelte-1jep0a:not(.rotated){transform:rotate(-90deg)}.accordion-body.svelte-1jep0a{padding:0 14px 14px}.trackio-loading.svelte-1kc6b2l{display:flex;align-items:center;justify-content:center;width:100%;min-height:min(70vh,640px);padding:32px 24px;box-sizing:border-box;background:transparent}.logo-stack.svelte-1kc6b2l{position:relative;width:min(100%,200px);max-width:min(92vw,200px);line-height:0;background:transparent;isolation:isolate}.logo-base.svelte-1kc6b2l{display:block;background:transparent}.logo-img.svelte-1kc6b2l{width:100%;height:auto;display:block;background:transparent}.logo-overlay.svelte-1kc6b2l{position:absolute;left:0;top:0;width:100%;animation:svelte-1kc6b2l-trackio-logo-sweep 4s linear infinite;pointer-events:none;background:transparent}.logo-overlay.svelte-1kc6b2l .logo-img:where(.svelte-1kc6b2l){width:100%;height:auto;object-position:left center}.logo-img--gray.svelte-1kc6b2l{filter:grayscale(1)}@keyframes svelte-1kc6b2l-trackio-logo-sweep{0%{clip-path:inset(0 0 0 0)}50%{clip-path:inset(0 0 0 100%)}to{clip-path:inset(0 0 0 0)}}@media(prefers-reduced-motion:reduce){.logo-overlay.svelte-1kc6b2l{display:none}}.sr-only.svelte-1kc6b2l{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.metrics-page.svelte-2bul55{padding:20px 24px;overflow-y:auto;flex:1;min-height:0}.plot-grid.svelte-2bul55{display:flex;flex-wrap:wrap;gap:16px}.subgroup-list.svelte-2bul55{margin-top:16px}.empty-state.svelte-2bul55{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-2bul55 h2:where(.svelte-2bul55){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-2bul55 p:where(.svelte-2bul55){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-2bul55 pre:where(.svelte-2bul55){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-2bul55 code:where(.svelte-2bul55){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-2bul55 pre:where(.svelte-2bul55) code:where(.svelte-2bul55){background:none;padding:0}.system-page.svelte-nv5os4{padding:20px 24px;overflow-y:auto;flex:1;min-height:0}.plot-grid.svelte-nv5os4{display:flex;flex-wrap:wrap;gap:12px}.empty-state.svelte-nv5os4{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-nv5os4 h2:where(.svelte-nv5os4){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-nv5os4 p:where(.svelte-nv5os4){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-nv5os4 pre:where(.svelte-nv5os4){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-nv5os4 ul:where(.svelte-nv5os4){list-style:disc;padding-left:20px;margin:4px 0 0}.empty-state.svelte-nv5os4 li:where(.svelte-nv5os4){margin:4px 0;color:var(--body-text-color, #1f2937)}.empty-state.svelte-nv5os4 code:where(.svelte-nv5os4){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-nv5os4 pre:where(.svelte-nv5os4) code:where(.svelte-nv5os4){background:none;padding:0}.table-container.svelte-1cp60rw{display:flex;flex-direction:column;gap:var(--size-2, 8px);position:relative}.header-row.svelte-1cp60rw{display:flex;justify-content:flex-end;align-items:center;min-height:var(--size-6, 24px);width:100%}.header-row.svelte-1cp60rw .label:where(.svelte-1cp60rw){flex:1;margin:0;color:var(--block-label-text-color, var(--neutral-500, #6b7280));font-size:var(--block-label-text-size, 12px);line-height:var(--line-sm, 1.4)}.table-wrap.svelte-1cp60rw{position:relative;overflow:auto;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--table-radius, var(--radius-lg, 8px))}table.svelte-1cp60rw{width:100%;table-layout:auto;color:var(--body-text-color, #1f2937);font-size:var(--input-text-size, 14px);line-height:var(--line-sm, 1.4);border-spacing:0;border-collapse:separate}thead.svelte-1cp60rw{position:sticky;top:0;z-index:5;box-shadow:var(--shadow-drop, rgba(0,0,0,.05) 0px 1px 2px 0px)}th.svelte-1cp60rw{padding:0;background:var(--table-even-background-fill, white);border-right-width:0px;border-left-width:1px;border-bottom-width:1px;border-style:solid;border-color:var(--border-color-primary, #e5e7eb);text-align:left;cursor:pointer;-webkit-user-select:none;user-select:none}th.first.svelte-1cp60rw{border-left-width:0;border-top-left-radius:var(--table-radius, var(--radius-lg, 8px))}th.last.svelte-1cp60rw{border-top-right-radius:var(--table-radius, var(--radius-lg, 8px))}.th-inner.svelte-1cp60rw{padding:var(--size-2, 8px);display:flex;align-items:center;gap:4px;font-weight:600;font-size:var(--text-sm, 12px);white-space:nowrap}.sort-arrow.svelte-1cp60rw{font-size:10px;color:var(--body-text-color-subdued, #9ca3af);visibility:hidden}.sort-arrow.visible.svelte-1cp60rw{visibility:visible}td.svelte-1cp60rw{padding:var(--size-2, 8px);border-right-width:0px;border-left-width:1px;border-bottom-width:1px;border-style:solid;border-color:var(--border-color-primary, #e5e7eb);font-size:var(--text-sm, 12px)}td.first.svelte-1cp60rw{border-left-width:0}tr.svelte-1cp60rw{background:var(--table-even-background-fill, white);border-bottom:1px solid var(--border-color-primary, #e5e7eb);text-align:left}tr.row-odd.svelte-1cp60rw{background:var(--table-odd-background-fill, var(--neutral-50, #f9fafb))}tr.selected.svelte-1cp60rw{background:var(--color-accent-soft, var(--primary-50, #fff7ed))}tr.svelte-1cp60rw:last-child td.first:where(.svelte-1cp60rw){border-bottom-left-radius:var(--table-radius, var(--radius-lg, 8px))}tr.svelte-1cp60rw:last-child td.last:where(.svelte-1cp60rw){border-bottom-right-radius:var(--table-radius, var(--radius-lg, 8px))}.check-col.svelte-1cp60rw{width:40px;text-align:center;padding:var(--size-2, 8px);border-left-width:0}.check-col.svelte-1cp60rw input[type=checkbox]:where(.svelte-1cp60rw){-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);cursor:pointer;flex-shrink:0;transition:background-color .15s,border-color .15s}.check-col.svelte-1cp60rw input[type=checkbox]:where(.svelte-1cp60rw):checked{background-image:var(--checkbox-check);background-color:var(--checkbox-background-color-selected, #2563eb);border-color:var(--checkbox-border-color-selected, #2563eb)}.media-page.svelte-outb32{padding:20px 24px;overflow-y:auto;flex:1}.section-title.svelte-outb32{font-size:var(--text-lg, 16px);font-weight:600;color:var(--body-text-color, #1f2937);margin:16px 0 8px}.gallery.svelte-outb32{display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:12px}.gallery-item.svelte-outb32{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);overflow:hidden;background:var(--background-fill-secondary, #f9fafb)}.gallery-item.svelte-outb32 img:where(.svelte-outb32),.gallery-item.svelte-outb32 video:where(.svelte-outb32){width:100%;display:block}.caption.svelte-outb32{padding:4px 8px;font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #9ca3af)}.step-label.svelte-outb32{padding:4px 8px;font-size:var(--text-xs, 10px);color:var(--body-text-color-subdued, #9ca3af)}.audio-list.svelte-outb32{display:flex;flex-direction:column;gap:8px}.audio-item.svelte-outb32{display:flex;align-items:center;gap:12px;padding:8px;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px)}.audio-label.svelte-outb32{font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #9ca3af);min-width:120px}.table-section.svelte-outb32{margin-bottom:16px}.empty-state.svelte-outb32{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-outb32 h2:where(.svelte-outb32){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-outb32 p:where(.svelte-outb32){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-outb32 pre:where(.svelte-outb32){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-outb32 code:where(.svelte-outb32){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-outb32 pre:where(.svelte-outb32) code:where(.svelte-outb32){background:none;padding:0}.reports-page.svelte-iufsej{padding:20px 24px;overflow-y:auto;flex:1}.controls.svelte-iufsej{display:flex;gap:16px;margin-bottom:16px;flex-wrap:wrap;align-items:flex-end}.control.svelte-iufsej{min-width:200px}.filter-pills.svelte-iufsej{display:flex;gap:4px}.pill.svelte-iufsej{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-xxl, 22px);padding:4px 12px;font-size:var(--text-sm, 12px);background:var(--background-fill-secondary, #f9fafb);color:var(--body-text-color-subdued, #6b7280);cursor:pointer;transition:background-color .15s,color .15s}.pill.svelte-iufsej:hover{background:var(--neutral-100, #f3f4f6)}.pill.active.svelte-iufsej{background:var(--color-accent, #f97316);color:#fff;border-color:var(--color-accent, #f97316)}.empty-state.svelte-iufsej{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-iufsej h2:where(.svelte-iufsej){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-iufsej p:where(.svelte-iufsej){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-iufsej pre:where(.svelte-iufsej){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-iufsej code:where(.svelte-iufsej){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-iufsej pre:where(.svelte-iufsej) code:where(.svelte-iufsej){background:none;padding:0}.alerts-table.svelte-iufsej{width:100%;border-collapse:collapse;font-size:var(--text-md, 14px)}.alerts-table.svelte-iufsej th:where(.svelte-iufsej){text-align:left;padding:8px 12px;border-bottom:2px solid var(--border-color-primary, #e5e7eb);color:var(--body-text-color-subdued, #6b7280);font-weight:600;font-size:var(--text-sm, 12px);text-transform:uppercase;letter-spacing:.05em}.alerts-table.svelte-iufsej td:where(.svelte-iufsej){padding:8px 12px;border-bottom:1px solid var(--border-color-primary, #e5e7eb);color:var(--body-text-color, #1f2937)}.alerts-table.svelte-iufsej tbody:where(.svelte-iufsej) tr:where(.svelte-iufsej):nth-child(odd){background:var(--table-odd-background-fill, var(--background-fill-primary, white))}.alerts-table.svelte-iufsej tbody:where(.svelte-iufsej) tr:where(.svelte-iufsej):nth-child(2n){background:var(--table-even-background-fill, var(--background-fill-secondary, #f9fafb))}.alerts-table.svelte-iufsej tr:where(.svelte-iufsej):hover{background:var(--background-fill-secondary, #f3f4f6)}.section-title.svelte-iufsej{font-size:16px;font-weight:700;margin:0 0 12px;color:var(--body-text-color, #1f2937)}.reports-section.svelte-iufsej,.alerts-section.svelte-iufsej{margin-bottom:32px}.report-card.svelte-iufsej{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);padding:16px 20px;margin-bottom:12px;background:var(--background-fill-primary, white)}.report-meta.svelte-iufsej{font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #6b7280);margin-bottom:8px}.report-content.svelte-iufsej{font-size:var(--text-md, 14px);color:var(--body-text-color, #1f2937);line-height:1.6}.report-content.svelte-iufsej h2{font-size:18px;font-weight:700;margin:0 0 8px}.report-content.svelte-iufsej h3{font-size:16px;font-weight:600;margin:12px 0 6px}.report-content.svelte-iufsej h4{font-size:14px;font-weight:600;margin:10px 0 4px}.report-content.svelte-iufsej code{background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.report-content.svelte-iufsej ul{margin:4px 0;padding-left:20px}.report-content.svelte-iufsej li{margin:2px 0}.report-content.svelte-iufsej p{margin:6px 0}.filter-empty.svelte-iufsej{color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-md, 14px)}.runs-page.svelte-1yb6d54{padding:20px 24px;overflow-y:auto;flex:1}.empty-state.svelte-1yb6d54{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-1yb6d54 h2:where(.svelte-1yb6d54){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-1yb6d54 p:where(.svelte-1yb6d54){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-1yb6d54 pre:where(.svelte-1yb6d54){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-1yb6d54 code:where(.svelte-1yb6d54){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-1yb6d54 pre:where(.svelte-1yb6d54) code:where(.svelte-1yb6d54){background:none;padding:0}.runs-table.svelte-1yb6d54{width:100%;border-collapse:collapse;font-size:var(--text-md, 14px)}.runs-table.svelte-1yb6d54 th:where(.svelte-1yb6d54){text-align:left;padding:8px 12px;border-bottom:2px solid var(--border-color-primary, #e5e7eb);color:var(--body-text-color-subdued, #6b7280);font-weight:600;font-size:var(--text-sm, 12px);text-transform:uppercase;letter-spacing:.05em}.runs-table.svelte-1yb6d54 td:where(.svelte-1yb6d54){padding:8px 12px;border-bottom:1px solid var(--border-color-primary, #e5e7eb);color:var(--body-text-color, #1f2937)}.runs-table.svelte-1yb6d54 tbody:where(.svelte-1yb6d54) tr:where(.svelte-1yb6d54):nth-child(odd){background:var(--table-odd-background-fill, var(--background-fill-primary, white))}.runs-table.svelte-1yb6d54 tbody:where(.svelte-1yb6d54) tr:where(.svelte-1yb6d54):nth-child(2n){background:var(--table-even-background-fill, var(--background-fill-secondary, #f9fafb))}.runs-table.svelte-1yb6d54 tr:where(.svelte-1yb6d54):hover{background:var(--background-fill-secondary, #f3f4f6)}.run-name-cell.svelte-1yb6d54{font-weight:500}.run-name-with-dot.svelte-1yb6d54{display:inline-flex;align-items:center;gap:8px;max-width:100%}.run-dot.svelte-1yb6d54{width:10px;height:10px;border-radius:50%;flex-shrink:0}.link-btn.svelte-1yb6d54{background:none;border:none;color:var(--color-accent, #f97316);cursor:pointer;font:inherit;font-weight:500;padding:0;text-align:left}.link-btn.svelte-1yb6d54:hover{text-decoration:underline}.rename-input.svelte-1yb6d54{font:inherit;padding:2px 6px;border:1px solid var(--color-accent, #f97316);border-radius:var(--radius-sm, 4px);outline:none;width:100%}.actions-cell.svelte-1yb6d54{display:flex;gap:4px}.action-btn.svelte-1yb6d54{background:none;border:1px solid transparent;color:var(--body-text-color-subdued, #6b7280);cursor:pointer;padding:4px;border-radius:var(--radius-sm, 4px);display:flex;align-items:center}.action-btn.svelte-1yb6d54:hover{background:var(--background-fill-secondary, #f9fafb);border-color:var(--border-color-primary, #e5e7eb);color:var(--body-text-color, #1f2937)}.delete-btn.svelte-1yb6d54:hover{color:#dc2626;border-color:#fecaca;background:#fef2f2}.action-btn.svelte-1yb6d54:disabled{opacity:.45;cursor:not-allowed;pointer-events:none}.run-detail-page.svelte-1bpgsx2{padding:20px 24px;overflow-y:auto;flex:1}.detail-card.svelte-1bpgsx2{background:var(--background-fill-primary, white);border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);padding:24px;max-width:800px}.detail-card.svelte-1bpgsx2 h2:where(.svelte-1bpgsx2){color:var(--body-text-color, #1f2937);margin:0 0 16px;font-size:var(--text-xl, 22px)}.detail-card.svelte-1bpgsx2 h3:where(.svelte-1bpgsx2){color:var(--body-text-color, #1f2937);margin:20px 0 8px;font-size:var(--text-lg, 16px)}.detail-grid.svelte-1bpgsx2{display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:12px}.detail-item.svelte-1bpgsx2{display:flex;flex-direction:column;gap:2px}.detail-label.svelte-1bpgsx2{font-size:var(--text-xs, 10px);font-weight:600;color:var(--body-text-color-subdued, #9ca3af);text-transform:uppercase}.detail-value.svelte-1bpgsx2{font-size:var(--text-md, 14px);color:var(--body-text-color, #1f2937)}.config-block.svelte-1bpgsx2{background:var(--background-fill-secondary, #f9fafb);padding:12px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:var(--text-sm, 12px);color:var(--body-text-color, #1f2937);overflow-x:auto}.empty-state.svelte-1bpgsx2{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-1bpgsx2 h2:where(.svelte-1bpgsx2){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-1bpgsx2 p:where(.svelte-1bpgsx2){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-1bpgsx2 pre:where(.svelte-1bpgsx2){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-1bpgsx2 code:where(.svelte-1bpgsx2){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-1bpgsx2 pre:where(.svelte-1bpgsx2) code:where(.svelte-1bpgsx2){background:none;padding:0}.files-page.svelte-1xvfk9n{padding:20px 24px;overflow-y:auto;flex:1}.page-title.svelte-1xvfk9n{color:var(--body-text-color, #1f2937);font-size:16px;font-weight:700;margin:0 0 4px}.page-subtitle.svelte-1xvfk9n{color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-sm, 12px);margin:0 0 16px}.file-list.svelte-1xvfk9n{display:flex;flex-direction:column;gap:4px}.file-item.svelte-1xvfk9n{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);background:var(--background-fill-primary, white);overflow:hidden}.file-item.expanded.svelte-1xvfk9n{border-color:var(--color-accent, #f97316)}.file-row.svelte-1xvfk9n{display:flex;align-items:center;justify-content:space-between;padding:10px 14px;gap:12px}.file-name.svelte-1xvfk9n{display:flex;align-items:center;gap:8px;background:none;border:none;padding:0;font-size:var(--text-md, 14px);color:var(--body-text-color, #1f2937);cursor:pointer;text-align:left}.file-name.svelte-1xvfk9n:hover{color:var(--color-accent, #f97316)}.file-icon.svelte-1xvfk9n{font-size:14px;flex-shrink:0}.file-actions.svelte-1xvfk9n{display:flex;align-items:center;gap:12px;flex-shrink:0}.file-size.svelte-1xvfk9n{font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #6b7280);white-space:nowrap}.download-btn.svelte-1xvfk9n{display:flex;align-items:center;justify-content:center;width:28px;height:28px;border-radius:var(--radius-md, 6px);color:var(--body-text-color-subdued, #6b7280);transition:background-color .15s,color .15s}.download-btn.svelte-1xvfk9n:hover{background:var(--background-fill-secondary, #f3f4f6);color:var(--body-text-color, #1f2937)}.file-preview.svelte-1xvfk9n{border-top:1px solid var(--border-color-primary, #e5e7eb);padding:12px 14px;background:var(--background-fill-secondary, #f9fafb)}.preview-code.svelte-1xvfk9n{margin:0;font-size:12px;line-height:1.5;max-height:400px;overflow:auto;white-space:pre-wrap;word-break:break-all;color:var(--body-text-color, #1f2937)}.preview-loading.svelte-1xvfk9n,.preview-unavailable.svelte-1xvfk9n{color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-sm, 12px);padding:8px 0}.preview-unavailable.svelte-1xvfk9n a:where(.svelte-1xvfk9n){color:var(--color-accent, #f97316);text-decoration:none}.preview-unavailable.svelte-1xvfk9n a:where(.svelte-1xvfk9n):hover{text-decoration:underline}.empty-state.svelte-1xvfk9n{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-1xvfk9n h2:where(.svelte-1xvfk9n){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-1xvfk9n p:where(.svelte-1xvfk9n){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-1xvfk9n pre:where(.svelte-1xvfk9n){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-1xvfk9n code:where(.svelte-1xvfk9n){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-1xvfk9n pre:where(.svelte-1xvfk9n) code:where(.svelte-1xvfk9n){background:none;padding:0}.settings-page.svelte-1ozf5k3{padding:24px 32px;overflow-y:auto;flex:1}.page-title.svelte-1ozf5k3{color:var(--body-text-color, #1f2937);font-size:18px;font-weight:700;margin:0 0 24px}.two-col.svelte-1ozf5k3{display:grid;grid-template-columns:minmax(0,1fr) minmax(0,1fr);gap:32px;align-items:start}@media(max-width:900px){.two-col.svelte-1ozf5k3{grid-template-columns:1fr}}.settings-section.svelte-1ozf5k3{margin-bottom:32px}.section-title.svelte-1ozf5k3{color:var(--body-text-color, #1f2937);font-size:15px;font-weight:600;margin:0 0 4px}.section-desc.svelte-1ozf5k3{color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-sm, 12px);margin:0 0 12px;line-height:1.5}.section-desc.svelte-1ozf5k3 code:where(.svelte-1ozf5k3){background:var(--background-fill-secondary, #f3f4f6);padding:1px 5px;border-radius:var(--radius-sm, 3px);font-size:11px}.section-desc.svelte-1ozf5k3 strong:where(.svelte-1ozf5k3){color:var(--color-accent, #f97316)}.theme-switcher.svelte-1ozf5k3{display:inline-flex;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);overflow:hidden}.theme-option.svelte-1ozf5k3{display:inline-flex;align-items:center;gap:6px;padding:8px 20px;border:none;background:var(--background-fill-primary, white);color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-md, 14px);cursor:pointer;transition:all .15s;border-right:1px solid var(--border-color-primary, #e5e7eb)}.theme-option.svelte-1ozf5k3:last-child{border-right:none}.theme-option.svelte-1ozf5k3:hover{color:var(--body-text-color, #1f2937);background:var(--background-fill-secondary, #f9fafb)}.theme-option.selected.svelte-1ozf5k3{background:var(--color-accent, #f97316);color:#fff;font-weight:500}.project-selector.svelte-1ozf5k3{display:flex;align-items:center;gap:10px;margin-bottom:12px}.selector-label.svelte-1ozf5k3{font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #6b7280);flex-shrink:0}.selector-select.svelte-1ozf5k3{padding:6px 10px;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-md, 4px);background:var(--background-fill-primary, white);color:var(--body-text-color, #1f2937);font-size:var(--text-sm, 12px);min-width:160px;cursor:pointer}.selector-select.svelte-1ozf5k3:focus{outline:none;border-color:var(--color-accent, #f97316)}.commands-table.svelte-1ozf5k3{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);overflow:hidden}.command-row.svelte-1ozf5k3{display:flex;align-items:center;gap:16px;padding:10px 14px;border-bottom:1px solid var(--border-color-primary, #e5e7eb)}.command-row.svelte-1ozf5k3:last-child{border-bottom:none}.command-label.svelte-1ozf5k3{width:180px;flex-shrink:0;font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #6b7280)}.command-value.svelte-1ozf5k3{flex:1;display:flex;align-items:center;gap:8px;min-width:0}.command-value.svelte-1ozf5k3 code:where(.svelte-1ozf5k3){flex:1;font-family:SFMono-Regular,Consolas,Liberation Mono,Menlo,monospace;font-size:12px;color:var(--body-text-color, #1f2937);white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.copy-btn.svelte-1ozf5k3{display:flex;align-items:center;justify-content:center;width:26px;height:26px;flex-shrink:0;border:none;background:none;border-radius:var(--radius-md, 4px);color:var(--body-text-color-subdued, #6b7280);cursor:pointer;transition:background-color .15s,color .15s}.copy-btn.svelte-1ozf5k3:hover{background:var(--background-fill-secondary, #f3f4f6);color:var(--body-text-color, #1f2937)}.copy-btn.copied.svelte-1ozf5k3{color:var(--color-accent, #f97316)}.agent-tabs.svelte-1ozf5k3{display:flex;border-bottom:1px solid var(--border-color-primary, #e5e7eb);gap:0;margin-bottom:0}.agent-tab.svelte-1ozf5k3{padding:8px 16px;border:none;background:none;color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-sm, 12px);cursor:pointer;border-bottom:2px solid transparent;transition:all .15s;white-space:nowrap}.agent-tab.svelte-1ozf5k3:hover{color:var(--body-text-color, #1f2937)}.agent-tab.active.svelte-1ozf5k3{color:var(--color-accent, #f97316);border-bottom-color:var(--color-accent, #f97316);font-weight:500}.agent-panel.svelte-1ozf5k3{border:1px solid var(--border-color-primary, #e5e7eb);border-top:none;border-radius:0 0 var(--radius-lg, 8px) var(--radius-lg, 8px);padding:16px}.install-block.svelte-1ozf5k3{margin-bottom:16px}.install-label.svelte-1ozf5k3{display:block;font-size:11px;font-weight:500;color:var(--body-text-color-subdued, #6b7280);text-transform:uppercase;letter-spacing:.04em;margin-bottom:6px}.install-cmd.svelte-1ozf5k3{display:flex;align-items:center;gap:8px;background:var(--background-fill-secondary, #f3f4f6);border-radius:var(--radius-md, 4px);padding:8px 10px}.install-cmd.svelte-1ozf5k3 code:where(.svelte-1ozf5k3){flex:1;font-family:SFMono-Regular,Consolas,Liberation Mono,Menlo,monospace;font-size:12px;color:var(--body-text-color, #1f2937);white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.example-block.svelte-1ozf5k3{background:var(--background-fill-secondary, #f9fafb);border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-md, 4px);padding:12px}.example-header.svelte-1ozf5k3{display:flex;align-items:center;justify-content:space-between;margin-bottom:8px}.example-label.svelte-1ozf5k3{font-size:11px;font-weight:500;color:var(--body-text-color-subdued, #6b7280);text-transform:uppercase;letter-spacing:.04em}.example-text.svelte-1ozf5k3{margin:0;font-size:var(--text-sm, 12px);color:var(--body-text-color, #1f2937);line-height:1.6;font-style:italic}*{margin:0;padding:0;box-sizing:border-box}body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,sans-serif;background:var(--background-fill-primary, #fff);color:var(--body-text-color, #1f2937);font-size:var(--text-md, 14px);-webkit-font-smoothing:antialiased}.app.svelte-1n46o8q{display:flex;height:100vh;overflow:hidden}.main.svelte-1n46o8q{flex:1;display:flex;flex-direction:column;overflow:hidden;min-width:0}.page-content.svelte-1n46o8q{flex:1;overflow:hidden;display:flex;background:var(--bg-primary)}
trackio/frontend/dist/assets/index-D_4A8v8I.js ADDED
The diff for this file is too large to render. See raw diff
 
trackio/frontend/dist/index.html ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
+ <title>Trackio Dashboard</title>
7
+ <link rel="icon" type="image/png" href="/static/trackio/trackio_logo_light.png" />
8
+ <script type="module" crossorigin src="/assets/index-D_4A8v8I.js"></script>
9
+ <link rel="stylesheet" crossorigin href="/assets/index-CicmrdCP.css">
10
+ </head>
11
+ <body>
12
+ <div id="app"></div>
13
+ </body>
14
+ </html>
trackio/frontend/eslint.config.js ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import js from "@eslint/js";
2
+ import svelte from "eslint-plugin-svelte";
3
+ import svelteParser from "svelte-eslint-parser";
4
+ import globals from "globals";
5
+
6
+ export default [
7
+ { ignores: ["dist/**", "node_modules/**"] },
8
+ {
9
+ files: ["**/*.js"],
10
+ languageOptions: {
11
+ globals: {
12
+ ...globals.browser,
13
+ ...globals.es2021,
14
+ $state: "readonly",
15
+ $derived: "readonly",
16
+ $effect: "readonly",
17
+ $props: "readonly",
18
+ $bindable: "readonly",
19
+ $inspect: "readonly",
20
+ },
21
+ },
22
+ rules: {
23
+ ...js.configs.recommended.rules,
24
+ "no-unused-vars": ["error", { argsIgnorePattern: "^_" }],
25
+ "no-empty": "off",
26
+ },
27
+ },
28
+ {
29
+ files: ["**/*.svelte"],
30
+ languageOptions: {
31
+ parser: svelteParser,
32
+ globals: { ...globals.browser, ...globals.es2021 },
33
+ },
34
+ plugins: { svelte },
35
+ rules: {
36
+ ...js.configs.recommended.rules,
37
+ ...svelte.configs.recommended.rules,
38
+ "no-unused-vars": ["error", { argsIgnorePattern: "^_", varsIgnorePattern: "^\\$" }],
39
+ "no-empty": "off",
40
+ },
41
+ },
42
+ ];
trackio/frontend/index.html ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
+ <title>Trackio Dashboard</title>
7
+ <link rel="icon" type="image/png" href="/static/trackio/trackio_logo_light.png" />
8
+ </head>
9
+ <body>
10
+ <div id="app"></div>
11
+ <script type="module" src="/src/main.js"></script>
12
+ </body>
13
+ </html>
trackio/frontend_server.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Serves the built Svelte frontend alongside the Trackio HTTP API."""
2
+
3
+ import logging
4
+ import re
5
+ from pathlib import Path
6
+
7
+ from starlette.responses import HTMLResponse
8
+ from starlette.routing import Mount, Route
9
+ from starlette.staticfiles import StaticFiles
10
+
11
+ FRONTEND_DIR = Path(__file__).parent / "frontend" / "dist"
12
+ ASSETS_DIR = Path(__file__).parent / "assets"
13
+
14
+ _logger = logging.getLogger(__name__)
15
+
16
+ _SPA_SEGMENTS = (
17
+ "metrics",
18
+ "system",
19
+ "media",
20
+ "reports",
21
+ "runs",
22
+ "run",
23
+ "files",
24
+ "settings",
25
+ )
26
+
27
+
28
+ def mount_frontend(app):
29
+ if not FRONTEND_DIR.exists():
30
+ _logger.warning(
31
+ "Trackio dashboard UI was not mounted: %s is missing. "
32
+ "Build the frontend with `npm ci && npm run build` in trackio/frontend.",
33
+ FRONTEND_DIR,
34
+ )
35
+ return
36
+
37
+ index_html_path = FRONTEND_DIR / "index.html"
38
+ if not index_html_path.exists():
39
+ _logger.warning(
40
+ "Trackio dashboard UI was not mounted: %s is missing.",
41
+ index_html_path,
42
+ )
43
+ return
44
+
45
+ index_html_content = index_html_path.read_text()
46
+ patched_html = re.sub(
47
+ r'/assets/(index-[^"]+)',
48
+ r"/assets/app/\1",
49
+ index_html_content,
50
+ )
51
+
52
+ async def serve_frontend(request):
53
+ return HTMLResponse(patched_html)
54
+
55
+ vite_assets = StaticFiles(directory=str(FRONTEND_DIR / "assets"))
56
+ static_assets = StaticFiles(directory=str(ASSETS_DIR))
57
+
58
+ app.routes.insert(0, Mount("/static/trackio", app=static_assets))
59
+ app.routes.insert(0, Mount("/assets/app", app=vite_assets))
60
+
61
+ for seg in reversed(_SPA_SEGMENTS):
62
+ app.routes.insert(0, Route(f"/{seg}/", serve_frontend, methods=["GET"]))
63
+ app.routes.insert(0, Route(f"/{seg}", serve_frontend, methods=["GET"]))
64
+ app.routes.insert(0, Route("/", serve_frontend, methods=["GET"]))
trackio/gpu.py ADDED
@@ -0,0 +1,357 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import threading
3
+ import warnings
4
+ from typing import TYPE_CHECKING, Any
5
+
6
+ if TYPE_CHECKING:
7
+ from trackio.run import Run
8
+
9
+ pynvml: Any = None
10
+ PYNVML_AVAILABLE = False
11
+ _nvml_initialized = False
12
+ _nvml_lock = threading.Lock()
13
+ _energy_baseline: dict[int, float] = {}
14
+
15
+
16
+ def _ensure_pynvml():
17
+ global PYNVML_AVAILABLE, pynvml
18
+ if PYNVML_AVAILABLE:
19
+ return pynvml
20
+ try:
21
+ import pynvml as _pynvml
22
+
23
+ pynvml = _pynvml
24
+ PYNVML_AVAILABLE = True
25
+ return pynvml
26
+ except ImportError:
27
+ raise ImportError(
28
+ "nvidia-ml-py is required for GPU monitoring. "
29
+ "Install it with: pip install nvidia-ml-py"
30
+ )
31
+
32
+
33
+ def _init_nvml() -> bool:
34
+ global _nvml_initialized
35
+ with _nvml_lock:
36
+ if _nvml_initialized:
37
+ return True
38
+ try:
39
+ nvml = _ensure_pynvml()
40
+ nvml.nvmlInit()
41
+ _nvml_initialized = True
42
+ return True
43
+ except Exception:
44
+ return False
45
+
46
+
47
+ def get_gpu_count() -> tuple[int, list[int]]:
48
+ """
49
+ Get the number of GPUs visible to this process and their physical indices.
50
+ Respects CUDA_VISIBLE_DEVICES environment variable.
51
+
52
+ Returns:
53
+ Tuple of (count, physical_indices) where:
54
+ - count: Number of visible GPUs
55
+ - physical_indices: List mapping logical index to physical GPU index.
56
+ e.g., if CUDA_VISIBLE_DEVICES=2,3 returns (2, [2, 3])
57
+ meaning logical GPU 0 = physical GPU 2, logical GPU 1 = physical GPU 3
58
+ """
59
+ if not _init_nvml():
60
+ return 0, []
61
+
62
+ cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES")
63
+ if cuda_visible is not None and cuda_visible.strip():
64
+ try:
65
+ indices = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()]
66
+ return len(indices), indices
67
+ except ValueError:
68
+ pass
69
+
70
+ try:
71
+ total = pynvml.nvmlDeviceGetCount()
72
+ return total, list(range(total))
73
+ except Exception:
74
+ return 0, []
75
+
76
+
77
+ def gpu_available() -> bool:
78
+ """
79
+ Check if GPU monitoring is available.
80
+
81
+ Returns True if nvidia-ml-py is installed and at least one NVIDIA GPU is detected.
82
+ This is used for auto-detection of GPU logging.
83
+ """
84
+ try:
85
+ _ensure_pynvml()
86
+ count, _ = get_gpu_count()
87
+ return count > 0
88
+ except ImportError:
89
+ return False
90
+ except Exception:
91
+ return False
92
+
93
+
94
+ def reset_energy_baseline():
95
+ """Reset the energy baseline for all GPUs. Called when a new run starts."""
96
+ global _energy_baseline
97
+ _energy_baseline = {}
98
+
99
+
100
+ def collect_gpu_metrics(device: int | None = None) -> dict:
101
+ """
102
+ Collect GPU metrics for visible GPUs.
103
+
104
+ Args:
105
+ device: CUDA device index to collect metrics from. If None, collects
106
+ from all GPUs visible to this process (respects CUDA_VISIBLE_DEVICES).
107
+ The device index is the logical CUDA index (0, 1, 2...), not the
108
+ physical GPU index.
109
+
110
+ Returns:
111
+ Dictionary of GPU metrics. Keys use logical device indices (gpu/0/, gpu/1/, etc.)
112
+ which correspond to CUDA device indices, not physical GPU indices.
113
+ """
114
+ if not _init_nvml():
115
+ return {}
116
+
117
+ gpu_count, visible_gpus = get_gpu_count()
118
+ if gpu_count == 0:
119
+ return {}
120
+
121
+ if device is not None:
122
+ if device < 0 or device >= gpu_count:
123
+ return {}
124
+ gpu_indices = [(device, visible_gpus[device])]
125
+ else:
126
+ gpu_indices = list(enumerate(visible_gpus))
127
+
128
+ metrics = {}
129
+ total_util = 0.0
130
+ total_mem_used_gib = 0.0
131
+ total_power = 0.0
132
+ max_temp = 0.0
133
+ valid_util_count = 0
134
+
135
+ for logical_idx, physical_idx in gpu_indices:
136
+ prefix = f"gpu/{logical_idx}"
137
+ try:
138
+ handle = pynvml.nvmlDeviceGetHandleByIndex(physical_idx)
139
+
140
+ try:
141
+ util = pynvml.nvmlDeviceGetUtilizationRates(handle)
142
+ metrics[f"{prefix}/utilization"] = util.gpu
143
+ metrics[f"{prefix}/memory_utilization"] = util.memory
144
+ total_util += util.gpu
145
+ valid_util_count += 1
146
+ except Exception:
147
+ pass
148
+
149
+ try:
150
+ mem = pynvml.nvmlDeviceGetMemoryInfo(handle)
151
+ mem_used_gib = mem.used / (1024**3)
152
+ mem_total_gib = mem.total / (1024**3)
153
+ metrics[f"{prefix}/allocated_memory"] = mem_used_gib
154
+ metrics[f"{prefix}/total_memory"] = mem_total_gib
155
+ if mem.total > 0:
156
+ metrics[f"{prefix}/memory_usage"] = mem.used / mem.total
157
+ total_mem_used_gib += mem_used_gib
158
+ except Exception:
159
+ pass
160
+
161
+ try:
162
+ power_mw = pynvml.nvmlDeviceGetPowerUsage(handle)
163
+ power_w = power_mw / 1000.0
164
+ metrics[f"{prefix}/power"] = power_w
165
+ total_power += power_w
166
+ except Exception:
167
+ pass
168
+
169
+ try:
170
+ power_limit_mw = pynvml.nvmlDeviceGetPowerManagementLimit(handle)
171
+ power_limit_w = power_limit_mw / 1000.0
172
+ metrics[f"{prefix}/power_limit"] = power_limit_w
173
+ if power_limit_w > 0 and f"{prefix}/power" in metrics:
174
+ metrics[f"{prefix}/power_percent"] = (
175
+ metrics[f"{prefix}/power"] / power_limit_w
176
+ ) * 100
177
+ except Exception:
178
+ pass
179
+
180
+ try:
181
+ temp = pynvml.nvmlDeviceGetTemperature(
182
+ handle, pynvml.NVML_TEMPERATURE_GPU
183
+ )
184
+ metrics[f"{prefix}/temp"] = temp
185
+ max_temp = max(max_temp, temp)
186
+ except Exception:
187
+ pass
188
+
189
+ try:
190
+ sm_clock = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_SM)
191
+ metrics[f"{prefix}/sm_clock"] = sm_clock
192
+ except Exception:
193
+ pass
194
+
195
+ try:
196
+ mem_clock = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_MEM)
197
+ metrics[f"{prefix}/memory_clock"] = mem_clock
198
+ except Exception:
199
+ pass
200
+
201
+ try:
202
+ fan_speed = pynvml.nvmlDeviceGetFanSpeed(handle)
203
+ metrics[f"{prefix}/fan_speed"] = fan_speed
204
+ except Exception:
205
+ pass
206
+
207
+ try:
208
+ pstate = pynvml.nvmlDeviceGetPerformanceState(handle)
209
+ metrics[f"{prefix}/performance_state"] = pstate
210
+ except Exception:
211
+ pass
212
+
213
+ try:
214
+ energy_mj = pynvml.nvmlDeviceGetTotalEnergyConsumption(handle)
215
+ if logical_idx not in _energy_baseline:
216
+ _energy_baseline[logical_idx] = energy_mj
217
+ energy_consumed_mj = energy_mj - _energy_baseline[logical_idx]
218
+ metrics[f"{prefix}/energy_consumed"] = energy_consumed_mj / 1000.0
219
+ except Exception:
220
+ pass
221
+
222
+ try:
223
+ pcie_tx = pynvml.nvmlDeviceGetPcieThroughput(
224
+ handle, pynvml.NVML_PCIE_UTIL_TX_BYTES
225
+ )
226
+ pcie_rx = pynvml.nvmlDeviceGetPcieThroughput(
227
+ handle, pynvml.NVML_PCIE_UTIL_RX_BYTES
228
+ )
229
+ metrics[f"{prefix}/pcie_tx"] = pcie_tx / 1024.0
230
+ metrics[f"{prefix}/pcie_rx"] = pcie_rx / 1024.0
231
+ except Exception:
232
+ pass
233
+
234
+ try:
235
+ throttle = pynvml.nvmlDeviceGetCurrentClocksThrottleReasons(handle)
236
+ metrics[f"{prefix}/throttle_thermal"] = int(
237
+ bool(throttle & pynvml.nvmlClocksThrottleReasonSwThermalSlowdown)
238
+ )
239
+ metrics[f"{prefix}/throttle_power"] = int(
240
+ bool(throttle & pynvml.nvmlClocksThrottleReasonSwPowerCap)
241
+ )
242
+ metrics[f"{prefix}/throttle_hw_slowdown"] = int(
243
+ bool(throttle & pynvml.nvmlClocksThrottleReasonHwSlowdown)
244
+ )
245
+ metrics[f"{prefix}/throttle_apps"] = int(
246
+ bool(
247
+ throttle
248
+ & pynvml.nvmlClocksThrottleReasonApplicationsClocksSetting
249
+ )
250
+ )
251
+ except Exception:
252
+ pass
253
+
254
+ try:
255
+ ecc_corrected = pynvml.nvmlDeviceGetTotalEccErrors(
256
+ handle,
257
+ pynvml.NVML_MEMORY_ERROR_TYPE_CORRECTED,
258
+ pynvml.NVML_VOLATILE_ECC,
259
+ )
260
+ metrics[f"{prefix}/corrected_memory_errors"] = ecc_corrected
261
+ except Exception:
262
+ pass
263
+
264
+ try:
265
+ ecc_uncorrected = pynvml.nvmlDeviceGetTotalEccErrors(
266
+ handle,
267
+ pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED,
268
+ pynvml.NVML_VOLATILE_ECC,
269
+ )
270
+ metrics[f"{prefix}/uncorrected_memory_errors"] = ecc_uncorrected
271
+ except Exception:
272
+ pass
273
+
274
+ except Exception:
275
+ continue
276
+
277
+ if valid_util_count > 0:
278
+ metrics["gpu/mean_utilization"] = total_util / valid_util_count
279
+ if total_mem_used_gib > 0:
280
+ metrics["gpu/total_allocated_memory"] = total_mem_used_gib
281
+ if total_power > 0:
282
+ metrics["gpu/total_power"] = total_power
283
+ if max_temp > 0:
284
+ metrics["gpu/max_temp"] = max_temp
285
+
286
+ return metrics
287
+
288
+
289
+ class GpuMonitor:
290
+ def __init__(self, run: "Run", interval: float = 10.0):
291
+ self._run = run
292
+ self._interval = interval
293
+ self._stop_flag = threading.Event()
294
+ self._thread: "threading.Thread | None" = None
295
+
296
+ def start(self):
297
+ count, _ = get_gpu_count()
298
+ if count == 0:
299
+ warnings.warn(
300
+ "auto_log_gpu=True but no NVIDIA GPUs detected. GPU logging disabled."
301
+ )
302
+ return
303
+
304
+ reset_energy_baseline()
305
+ self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
306
+ self._thread.start()
307
+
308
+ def stop(self):
309
+ self._stop_flag.set()
310
+ if self._thread is not None:
311
+ self._thread.join(timeout=2.0)
312
+
313
+ def _monitor_loop(self):
314
+ while not self._stop_flag.is_set():
315
+ try:
316
+ metrics = collect_gpu_metrics()
317
+ if metrics:
318
+ self._run.log_system(metrics)
319
+ except Exception:
320
+ pass
321
+
322
+ self._stop_flag.wait(timeout=self._interval)
323
+
324
+
325
+ def log_gpu(run: "Run | None" = None, device: int | None = None) -> dict:
326
+ """
327
+ Log GPU metrics to the current or specified run as system metrics.
328
+
329
+ Args:
330
+ run: Optional Run instance. If None, uses current run from context.
331
+ device: CUDA device index to collect metrics from. If None, collects
332
+ from all GPUs visible to this process (respects CUDA_VISIBLE_DEVICES).
333
+
334
+ Returns:
335
+ dict: The GPU metrics that were logged.
336
+
337
+ Example:
338
+ ```python
339
+ import trackio
340
+
341
+ run = trackio.init(project="my-project")
342
+ trackio.log({"loss": 0.5})
343
+ trackio.log_gpu() # logs all visible GPUs
344
+ trackio.log_gpu(device=0) # logs only CUDA device 0
345
+ ```
346
+ """
347
+ from trackio import context_vars
348
+
349
+ if run is None:
350
+ run = context_vars.current_run.get()
351
+ if run is None:
352
+ raise RuntimeError("Call trackio.init() before trackio.log_gpu().")
353
+
354
+ metrics = collect_gpu_metrics(device=device)
355
+ if metrics:
356
+ run.log_system(metrics)
357
+ return metrics
trackio/histogram.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Sequence
2
+
3
+ import numpy as np
4
+
5
+
6
+ class Histogram:
7
+ """
8
+ Histogram data type for Trackio, compatible with wandb.Histogram.
9
+
10
+ Args:
11
+ sequence (`np.ndarray` or `Sequence[float]` or `Sequence[int]`, *optional*):
12
+ Sequence of values to create the histogram from.
13
+ np_histogram (`tuple`, *optional*):
14
+ Pre-computed NumPy histogram as a `(hist, bins)` tuple.
15
+ num_bins (`int`, *optional*, defaults to `64`):
16
+ Number of bins for the histogram (maximum `512`).
17
+
18
+ Example:
19
+ ```python
20
+ import trackio
21
+ import numpy as np
22
+
23
+ # Create histogram from sequence
24
+ data = np.random.randn(1000)
25
+ trackio.log({"distribution": trackio.Histogram(data)})
26
+
27
+ # Create histogram from numpy histogram
28
+ hist, bins = np.histogram(data, bins=30)
29
+ trackio.log({"distribution": trackio.Histogram(np_histogram=(hist, bins))})
30
+
31
+ # Specify custom number of bins
32
+ trackio.log({"distribution": trackio.Histogram(data, num_bins=50)})
33
+ ```
34
+ """
35
+
36
+ TYPE = "trackio.histogram"
37
+
38
+ def __init__(
39
+ self,
40
+ sequence: np.ndarray | Sequence[float] | Sequence[int] | None = None,
41
+ np_histogram: tuple | None = None,
42
+ num_bins: int = 64,
43
+ ):
44
+ if sequence is None and np_histogram is None:
45
+ raise ValueError("Must provide either sequence or np_histogram")
46
+
47
+ if sequence is not None and np_histogram is not None:
48
+ raise ValueError("Cannot provide both sequence and np_histogram")
49
+
50
+ num_bins = min(num_bins, 512)
51
+
52
+ if np_histogram is not None:
53
+ self.histogram, self.bins = np_histogram
54
+ self.histogram = np.asarray(self.histogram)
55
+ self.bins = np.asarray(self.bins)
56
+ else:
57
+ data = np.asarray(sequence).flatten()
58
+ data = data[np.isfinite(data)]
59
+ if len(data) == 0:
60
+ self.histogram = np.array([])
61
+ self.bins = np.array([])
62
+ else:
63
+ self.histogram, self.bins = np.histogram(data, bins=num_bins)
64
+
65
+ def _to_dict(self) -> dict:
66
+ """Convert histogram to dictionary for storage."""
67
+ return {
68
+ "_type": self.TYPE,
69
+ "bins": self.bins.tolist(),
70
+ "values": self.histogram.tolist(),
71
+ }
trackio/imports.py ADDED
@@ -0,0 +1,300 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import csv
2
+ import os
3
+ from pathlib import Path
4
+
5
+ from trackio import deploy, utils
6
+ from trackio.sqlite_storage import SQLiteStorage
7
+
8
+
9
+ def import_csv(
10
+ csv_path: str | Path,
11
+ project: str,
12
+ name: str | None = None,
13
+ space_id: str | None = None,
14
+ dataset_id: str | None = None,
15
+ private: bool | None = None,
16
+ force: bool = False,
17
+ ) -> None:
18
+ """
19
+ Imports a CSV file into a Trackio project. The CSV file must contain a `"step"`
20
+ column, may optionally contain a `"timestamp"` column, and any other columns will be
21
+ treated as metrics. It should also include a header row with the column names.
22
+
23
+ TODO: call init() and return a Run object so that the user can continue to log metrics to it.
24
+
25
+ Args:
26
+ csv_path (`str` or `Path`):
27
+ The str or Path to the CSV file to import.
28
+ project (`str`):
29
+ The name of the project to import the CSV file into. Must not be an existing
30
+ project.
31
+ name (`str`, *optional*):
32
+ The name of the Run to import the CSV file into. If not provided, a default
33
+ name will be generated.
34
+ name (`str`, *optional*):
35
+ The name of the run (if not provided, a default name will be generated).
36
+ space_id (`str`, *optional*):
37
+ If provided, the project will be logged to a Hugging Face Space instead of a
38
+ local directory. Should be a complete Space name like `"username/reponame"`
39
+ or `"orgname/reponame"`, or just `"reponame"` in which case the Space will
40
+ be created in the currently-logged-in Hugging Face user's namespace. If the
41
+ Space does not exist, it will be created. If the Space already exists, the
42
+ project will be logged to it.
43
+ dataset_id (`str`, *optional*):
44
+ Deprecated. Use `bucket_id` instead.
45
+ private (`bool`, *optional*):
46
+ Whether to make the Space private. If None (default), the repo will be
47
+ public unless the organization's default is private. This value is ignored
48
+ if the repo already exists.
49
+ """
50
+ if SQLiteStorage.get_runs(project):
51
+ raise ValueError(
52
+ f"Project '{project}' already exists. Cannot import CSV into existing project."
53
+ )
54
+
55
+ csv_path = Path(csv_path)
56
+ if not csv_path.exists():
57
+ raise FileNotFoundError(f"CSV file not found: {csv_path}")
58
+
59
+ with csv_path.open(newline="", encoding="utf-8") as csv_file:
60
+ reader = csv.DictReader(csv_file)
61
+ source_columns = reader.fieldnames or []
62
+ rows = list(reader)
63
+
64
+ if not rows:
65
+ raise ValueError("CSV file is empty")
66
+
67
+ column_mapping = utils.simplify_column_names(source_columns)
68
+ normalized_rows = [
69
+ {column_mapping[key]: value for key, value in row.items()} for row in rows
70
+ ]
71
+ columns = list(normalized_rows[0].keys())
72
+
73
+ step_column = None
74
+ for col in columns:
75
+ if col.lower() == "step":
76
+ step_column = col
77
+ break
78
+
79
+ if step_column is None:
80
+ raise ValueError("CSV file must contain a 'step' or 'Step' column")
81
+
82
+ if name is None:
83
+ name = csv_path.stem
84
+
85
+ metrics_list = []
86
+ steps = []
87
+ timestamps = []
88
+
89
+ numeric_columns = []
90
+ for column in columns:
91
+ if column == step_column:
92
+ continue
93
+ if column == "timestamp":
94
+ continue
95
+
96
+ try:
97
+ for row in normalized_rows:
98
+ value = row[column]
99
+ if value in ("", None):
100
+ continue
101
+ float(value)
102
+ except (ValueError, TypeError):
103
+ continue
104
+ numeric_columns.append(column)
105
+
106
+ for row in normalized_rows:
107
+ metrics = {}
108
+ for column in numeric_columns:
109
+ value = row[column]
110
+ if value not in ("", None):
111
+ metrics[column] = float(value)
112
+
113
+ if metrics:
114
+ metrics_list.append(metrics)
115
+ steps.append(int(float(row[step_column])))
116
+
117
+ if "timestamp" in row and row["timestamp"] not in ("", None):
118
+ timestamps.append(str(row["timestamp"]))
119
+ else:
120
+ timestamps.append("")
121
+
122
+ if metrics_list:
123
+ SQLiteStorage.bulk_log(
124
+ project=project,
125
+ run=name,
126
+ metrics_list=metrics_list,
127
+ steps=steps,
128
+ timestamps=timestamps,
129
+ )
130
+
131
+ print(
132
+ f"* Imported {len(metrics_list)} rows from {csv_path} into project '{project}' as run '{name}'"
133
+ )
134
+ print(f"* Metrics found: {', '.join(metrics_list[0].keys())}")
135
+
136
+ space_id, dataset_id, _ = utils.preprocess_space_and_dataset_ids(
137
+ space_id, dataset_id
138
+ )
139
+ if dataset_id is not None:
140
+ os.environ["TRACKIO_DATASET_ID"] = dataset_id
141
+ print(f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}")
142
+
143
+ if space_id is None:
144
+ utils.print_dashboard_instructions(project)
145
+ else:
146
+ deploy.create_space_if_not_exists(
147
+ space_id=space_id, dataset_id=dataset_id, private=private
148
+ )
149
+ deploy.wait_until_space_exists(space_id=space_id)
150
+ deploy.upload_db_to_space(project=project, space_id=space_id, force=force)
151
+ print(
152
+ f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}"
153
+ )
154
+
155
+
156
+ def import_tf_events(
157
+ log_dir: str | Path,
158
+ project: str,
159
+ name: str | None = None,
160
+ space_id: str | None = None,
161
+ dataset_id: str | None = None,
162
+ private: bool | None = None,
163
+ force: bool = False,
164
+ ) -> None:
165
+ """
166
+ Imports TensorFlow Events files from a directory into a Trackio project. Each
167
+ subdirectory in the log directory will be imported as a separate run.
168
+
169
+ Args:
170
+ log_dir (`str` or `Path`):
171
+ The str or Path to the directory containing TensorFlow Events files.
172
+ project (`str`):
173
+ The name of the project to import the TensorFlow Events files into. Must not
174
+ be an existing project.
175
+ name (`str`, *optional*):
176
+ The name prefix for runs (if not provided, will use directory names). Each
177
+ subdirectory will create a separate run.
178
+ space_id (`str`, *optional*):
179
+ If provided, the project will be logged to a Hugging Face Space instead of a
180
+ local directory. Should be a complete Space name like `"username/reponame"`
181
+ or `"orgname/reponame"`, or just `"reponame"` in which case the Space will
182
+ be created in the currently-logged-in Hugging Face user's namespace. If the
183
+ Space does not exist, it will be created. If the Space already exists, the
184
+ project will be logged to it.
185
+ dataset_id (`str`, *optional*):
186
+ Deprecated. Use `bucket_id` instead.
187
+ private (`bool`, *optional*):
188
+ Whether to make the Space private. If None (default), the repo will be
189
+ public unless the organization's default is private. This value is ignored
190
+ if the repo already exists.
191
+ """
192
+ try:
193
+ from tbparse import SummaryReader
194
+ except ImportError:
195
+ raise ImportError(
196
+ "The `tbparse` package is not installed but is required for `import_tf_events`. Please install trackio with the `tensorboard` extra: `pip install trackio[tensorboard]`."
197
+ )
198
+
199
+ if SQLiteStorage.get_runs(project):
200
+ raise ValueError(
201
+ f"Project '{project}' already exists. Cannot import TF events into existing project."
202
+ )
203
+
204
+ path = Path(log_dir)
205
+ if not path.exists():
206
+ raise FileNotFoundError(f"TF events directory not found: {path}")
207
+
208
+ # Use tbparse to read all tfevents files in the directory structure
209
+ reader = SummaryReader(str(path), extra_columns={"dir_name"})
210
+ df = reader.scalars
211
+
212
+ if df.empty:
213
+ raise ValueError(f"No TensorFlow events data found in {path}")
214
+
215
+ total_imported = 0
216
+ imported_runs = []
217
+
218
+ # Group by dir_name to create separate runs
219
+ for dir_name, group_df in df.groupby("dir_name"):
220
+ try:
221
+ # Determine run name based on directory name
222
+ if dir_name == "":
223
+ run_name = "main" # For files in the root directory
224
+ else:
225
+ run_name = dir_name # Use directory name
226
+
227
+ if name:
228
+ run_name = f"{name}_{run_name}"
229
+
230
+ if group_df.empty:
231
+ print(f"* Skipping directory {dir_name}: no scalar data found")
232
+ continue
233
+
234
+ metrics_list = []
235
+ steps = []
236
+ timestamps = []
237
+
238
+ for _, row in group_df.iterrows():
239
+ # Convert row values to appropriate types
240
+ tag = str(row["tag"])
241
+ value = float(row["value"])
242
+ step = int(row["step"])
243
+
244
+ metrics = {tag: value}
245
+ metrics_list.append(metrics)
246
+ steps.append(step)
247
+
248
+ # Use wall_time if present, else fallback
249
+ if "wall_time" in group_df.columns and not utils.is_missing_value(
250
+ row["wall_time"]
251
+ ):
252
+ timestamps.append(str(row["wall_time"]))
253
+ else:
254
+ timestamps.append("")
255
+
256
+ if metrics_list:
257
+ SQLiteStorage.bulk_log(
258
+ project=project,
259
+ run=str(run_name),
260
+ metrics_list=metrics_list,
261
+ steps=steps,
262
+ timestamps=timestamps,
263
+ )
264
+
265
+ total_imported += len(metrics_list)
266
+ imported_runs.append(run_name)
267
+
268
+ print(
269
+ f"* Imported {len(metrics_list)} scalar events from directory '{dir_name}' as run '{run_name}'"
270
+ )
271
+ print(f"* Metrics in this run: {', '.join(set(group_df['tag']))}")
272
+
273
+ except Exception as e:
274
+ print(f"* Error processing directory {dir_name}: {e}")
275
+ continue
276
+
277
+ if not imported_runs:
278
+ raise ValueError("No valid TensorFlow events data could be imported")
279
+
280
+ print(f"* Total imported events: {total_imported}")
281
+ print(f"* Created runs: {', '.join(imported_runs)}")
282
+
283
+ space_id, dataset_id, _ = utils.preprocess_space_and_dataset_ids(
284
+ space_id, dataset_id
285
+ )
286
+ if dataset_id is not None:
287
+ os.environ["TRACKIO_DATASET_ID"] = dataset_id
288
+ print(f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}")
289
+
290
+ if space_id is None:
291
+ utils.print_dashboard_instructions(project)
292
+ else:
293
+ deploy.create_space_if_not_exists(
294
+ space_id, dataset_id=dataset_id, private=private
295
+ )
296
+ deploy.wait_until_space_exists(space_id)
297
+ deploy.upload_db_to_space(project, space_id, force=force)
298
+ print(
299
+ f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}"
300
+ )
trackio/launch.py ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import secrets
5
+ import socket
6
+ import threading
7
+ import time
8
+ import warnings
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import httpx
13
+ import uvicorn
14
+ from uvicorn.config import Config
15
+
16
+ from trackio._vendor.gradio_exceptions import ChecksumMismatchError
17
+ from trackio._vendor.networking import normalize_share_url, setup_tunnel, url_ok
18
+ from trackio._vendor.tunneling import BINARY_PATH
19
+ from trackio.launch_utils import colab_check, is_hosted_notebook
20
+
21
+ INITIAL_PORT_VALUE = int(os.getenv("GRADIO_SERVER_PORT", "7860"))
22
+ TRY_NUM_PORTS = int(os.getenv("GRADIO_NUM_PORTS", "100"))
23
+ LOCALHOST_NAME = os.getenv("GRADIO_SERVER_NAME", "127.0.0.1")
24
+
25
+
26
+ class _UvicornServer(uvicorn.Server):
27
+ def install_signal_handlers(self) -> None:
28
+ pass
29
+
30
+ def run_in_thread(self) -> None:
31
+ self.thread = threading.Thread(target=self.run, daemon=True)
32
+ self.thread.start()
33
+ start = time.time()
34
+ while not self.started:
35
+ time.sleep(1e-3)
36
+ if time.time() - start > 60:
37
+ raise RuntimeError(
38
+ "Server failed to start. Please check that the port is available."
39
+ )
40
+
41
+
42
+ def _bind_host(server_name: str) -> str:
43
+ if server_name.startswith("[") and server_name.endswith("]"):
44
+ return server_name[1:-1]
45
+ return server_name
46
+
47
+
48
+ def start_server(
49
+ app: Any,
50
+ server_name: str | None = None,
51
+ server_port: int | None = None,
52
+ ssl_keyfile: str | None = None,
53
+ ssl_certfile: str | None = None,
54
+ ssl_keyfile_password: str | None = None,
55
+ ) -> tuple[str, int, str, _UvicornServer]:
56
+ server_name = server_name or LOCALHOST_NAME
57
+ url_host_name = "localhost" if server_name == "0.0.0.0" else server_name
58
+
59
+ host = _bind_host(server_name)
60
+
61
+ server_ports = (
62
+ [server_port]
63
+ if server_port is not None
64
+ else range(INITIAL_PORT_VALUE, INITIAL_PORT_VALUE + TRY_NUM_PORTS)
65
+ )
66
+
67
+ port_used = None
68
+ server = None
69
+ for port in server_ports:
70
+ try:
71
+ socket_family = socket.AF_INET6 if ":" in host else socket.AF_INET
72
+ with socket.socket(socket_family, socket.SOCK_STREAM) as s:
73
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
74
+ s.bind((host, port))
75
+ config = Config(
76
+ app=app,
77
+ port=port,
78
+ host=host,
79
+ log_level="warning",
80
+ ssl_keyfile=ssl_keyfile,
81
+ ssl_certfile=ssl_certfile,
82
+ ssl_keyfile_password=ssl_keyfile_password,
83
+ )
84
+ server = _UvicornServer(config=config)
85
+ server.run_in_thread()
86
+ port_used = port
87
+ break
88
+ except (OSError, RuntimeError):
89
+ continue
90
+ else:
91
+ raise OSError(
92
+ f"Cannot find empty port in range: {min(server_ports)}-{max(server_ports)}. "
93
+ "Set GRADIO_SERVER_PORT or pass server_port to trackio.show()."
94
+ )
95
+
96
+ assert port_used is not None and server is not None
97
+
98
+ if ssl_keyfile is not None:
99
+ path_to_local_server = f"https://{url_host_name}:{port_used}/"
100
+ else:
101
+ path_to_local_server = f"http://{url_host_name}:{port_used}/"
102
+
103
+ return server_name, port_used, path_to_local_server, server
104
+
105
+
106
+ def launch_trackio_dashboard(
107
+ starlette_app: Any,
108
+ *,
109
+ server_name: str | None = None,
110
+ server_port: int | None = None,
111
+ share: bool | None = None,
112
+ share_server_address: str | None = None,
113
+ share_server_protocol: str | None = None,
114
+ share_server_tls_certificate: str | None = None,
115
+ mcp_server: bool = False,
116
+ ssl_verify: bool = True,
117
+ quiet: bool = False,
118
+ ) -> tuple[str | None, str | None, str | None, Any]:
119
+ is_colab = colab_check()
120
+ is_hosted_nb = is_hosted_notebook()
121
+ space_id = os.getenv("SPACE_ID")
122
+
123
+ if share is None:
124
+ if is_colab or is_hosted_nb:
125
+ if not quiet:
126
+ print(
127
+ "It looks like you are running Trackio on a hosted Jupyter notebook, which requires "
128
+ "`share=True`. Automatically setting `share=True` "
129
+ "(set `share=False` in `show()` to disable).\n"
130
+ )
131
+ share = True
132
+ else:
133
+ share = os.getenv("GRADIO_SHARE", "").lower() == "true"
134
+
135
+ sn = server_name
136
+ if sn is None and os.getenv("SYSTEM") == "spaces":
137
+ sn = "0.0.0.0"
138
+ elif sn is None:
139
+ sn = LOCALHOST_NAME
140
+
141
+ server_name_r, server_port_r, local_url, uv_server = start_server(
142
+ starlette_app,
143
+ server_name=sn,
144
+ server_port=server_port,
145
+ )
146
+
147
+ local_api_url = f"{local_url.rstrip('/')}/api/"
148
+ try:
149
+ httpx.get(f"{local_url.rstrip('/')}/version", verify=ssl_verify, timeout=10)
150
+ except Exception as e:
151
+ raise RuntimeError(
152
+ f"Could not reach Trackio server at {local_url.rstrip('/')}/version: {e}"
153
+ ) from e
154
+
155
+ if share and space_id:
156
+ warnings.warn("Setting share=True is not supported on Hugging Face Spaces")
157
+ share = False
158
+
159
+ share_url: str | None = None
160
+ if share:
161
+ try:
162
+ share_tok = secrets.token_urlsafe(32)
163
+ proto = share_server_protocol or (
164
+ "http" if share_server_address is not None else "https"
165
+ )
166
+ raw = setup_tunnel(
167
+ local_host=server_name_r,
168
+ local_port=server_port_r,
169
+ share_token=share_tok,
170
+ share_server_address=share_server_address,
171
+ share_server_tls_certificate=share_server_tls_certificate,
172
+ )
173
+ share_url = normalize_share_url(raw, proto)
174
+ if not quiet:
175
+ print(f"* Running on public URL: {share_url}")
176
+ print(
177
+ "\nThis share link expires in 1 week. For permanent hosting, deploy to Hugging Face Spaces."
178
+ )
179
+ except Exception as e:
180
+ share_url = None
181
+ if not quiet:
182
+ if isinstance(e, ChecksumMismatchError):
183
+ print(
184
+ "\nCould not create share link. Checksum mismatch for frpc binary."
185
+ )
186
+ elif Path(BINARY_PATH).exists():
187
+ print(
188
+ "\nCould not create share link. Check your internet connection or https://status.gradio.app."
189
+ )
190
+ else:
191
+ print(
192
+ f"\nCould not create share link. Missing frpc at {BINARY_PATH}. {e}"
193
+ )
194
+
195
+ if not share_url and not quiet:
196
+ print("* To create a public link, set `share=True` in `trackio.show()`.")
197
+
198
+ return local_url, share_url, local_api_url, uv_server
199
+
200
+
201
+ def url_ok_local(local_url: str) -> bool:
202
+ return url_ok(local_url)
trackio/launch_utils.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+
4
+ def colab_check() -> bool:
5
+ is_colab = False
6
+ try:
7
+ from IPython.core.getipython import get_ipython # noqa: PLC0415
8
+
9
+ from_ipynb = get_ipython()
10
+ if "google.colab" in str(from_ipynb):
11
+ is_colab = True
12
+ except (ImportError, NameError):
13
+ pass
14
+ return is_colab
15
+
16
+
17
+ def is_hosted_notebook() -> bool:
18
+ return bool(
19
+ os.environ.get("KAGGLE_KERNEL_RUN_TYPE")
20
+ or os.path.exists("/home/ec2-user/SageMaker")
21
+ )
22
+
23
+
24
+ def ipython_check() -> bool:
25
+ is_ipython = False
26
+ try:
27
+ from IPython.core.getipython import get_ipython # noqa: PLC0415
28
+
29
+ if get_ipython() is not None:
30
+ is_ipython = True
31
+ except (ImportError, NameError):
32
+ pass
33
+ return is_ipython
trackio/markdown.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ class Markdown:
2
+ """
3
+ Markdown report data type for Trackio.
4
+
5
+ Args:
6
+ text (`str`):
7
+ Markdown content to log.
8
+ """
9
+
10
+ TYPE = "trackio.markdown"
11
+
12
+ def __init__(self, text: str = ""):
13
+ if not isinstance(text, str):
14
+ raise ValueError("Markdown text must be a string")
15
+ self.text = text
16
+
17
+ def _to_dict(self) -> dict:
18
+ return {
19
+ "_type": self.TYPE,
20
+ "_value": self.text,
21
+ }
trackio/mcp_setup.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import secrets
5
+ from contextlib import asynccontextmanager
6
+ from typing import Any
7
+
8
+ from starlette.routing import Mount
9
+
10
+ from trackio.sqlite_storage import SQLiteStorage
11
+
12
+
13
+ def _assert_mcp_mutation_access(
14
+ *,
15
+ hf_token: str | None = None,
16
+ write_token: str | None = None,
17
+ ) -> None:
18
+ import trackio.server as trackio_server # noqa: PLC0415
19
+
20
+ if os.getenv("SYSTEM") == "spaces":
21
+ try:
22
+ trackio_server.check_hf_token_has_write_access(hf_token)
23
+ except PermissionError as e:
24
+ raise ValueError(str(e)) from e
25
+ return
26
+
27
+ if not secrets.compare_digest(write_token or "", trackio_server.write_token or ""):
28
+ raise ValueError(
29
+ "A write_token is required for Trackio MCP mutations. "
30
+ "Use the write token from the dashboard URL."
31
+ )
32
+
33
+
34
+ def create_mcp_integration() -> tuple[list[Any], Any]:
35
+ from mcp.server.fastmcp import FastMCP # noqa: PLC0415
36
+
37
+ import trackio.server as trackio_server # noqa: PLC0415
38
+
39
+ mcp = FastMCP(
40
+ "Trackio",
41
+ instructions="Inspect and manage Trackio experiment data.",
42
+ streamable_http_path="/",
43
+ log_level="WARNING",
44
+ )
45
+
46
+ mcp.add_tool(
47
+ trackio_server.get_all_projects,
48
+ description="List all Trackio projects available on this server.",
49
+ structured_output=True,
50
+ )
51
+ mcp.add_tool(
52
+ trackio_server.get_runs_for_project,
53
+ description="List runs for a given Trackio project.",
54
+ structured_output=True,
55
+ )
56
+ mcp.add_tool(
57
+ trackio_server.get_metrics_for_run,
58
+ description="List metric names recorded for a given Trackio run.",
59
+ structured_output=True,
60
+ )
61
+ mcp.add_tool(
62
+ trackio_server.get_project_summary,
63
+ description="Return summary metadata for a Trackio project.",
64
+ structured_output=True,
65
+ )
66
+ mcp.add_tool(
67
+ trackio_server.get_run_summary,
68
+ description="Return summary metadata for a Trackio run.",
69
+ structured_output=True,
70
+ )
71
+ mcp.add_tool(
72
+ trackio_server.get_metric_values,
73
+ description="Fetch metric values for a run, optionally around a step or time.",
74
+ structured_output=True,
75
+ )
76
+ mcp.add_tool(
77
+ trackio_server.get_system_metrics_for_run,
78
+ description="List system metric names recorded for a run.",
79
+ structured_output=True,
80
+ )
81
+ mcp.add_tool(
82
+ trackio_server.get_system_logs,
83
+ description="Fetch system metric logs for a run.",
84
+ structured_output=True,
85
+ )
86
+ mcp.add_tool(
87
+ trackio_server.get_snapshot,
88
+ description="Fetch a single Trackio snapshot around a step or timestamp.",
89
+ structured_output=True,
90
+ )
91
+ mcp.add_tool(
92
+ trackio_server.get_logs,
93
+ description="Fetch Trackio metric logs for a run.",
94
+ structured_output=True,
95
+ )
96
+ mcp.add_tool(
97
+ trackio_server.get_alerts,
98
+ description="Fetch alerts for a project, optionally filtered by run or level.",
99
+ structured_output=True,
100
+ )
101
+ mcp.add_tool(
102
+ trackio_server.get_settings,
103
+ description="Return Trackio dashboard settings and asset configuration.",
104
+ structured_output=True,
105
+ )
106
+
107
+ @mcp.tool(
108
+ description="Delete a run. On Spaces, pass an hf_token with write access.",
109
+ structured_output=True,
110
+ )
111
+ def delete_run(
112
+ project: str,
113
+ run: str,
114
+ hf_token: str | None = None,
115
+ write_token: str | None = None,
116
+ ) -> bool:
117
+ _assert_mcp_mutation_access(hf_token=hf_token, write_token=write_token)
118
+ return SQLiteStorage.delete_run(project, run)
119
+
120
+ @mcp.tool(
121
+ description="Rename a run. On Spaces, pass an hf_token with write access.",
122
+ structured_output=True,
123
+ )
124
+ def rename_run(
125
+ project: str,
126
+ old_name: str,
127
+ new_name: str,
128
+ hf_token: str | None = None,
129
+ write_token: str | None = None,
130
+ ) -> bool:
131
+ _assert_mcp_mutation_access(hf_token=hf_token, write_token=write_token)
132
+ SQLiteStorage.rename_run(project, old_name, new_name)
133
+ return True
134
+
135
+ @mcp.tool(
136
+ description=(
137
+ "Trigger a Trackio export/sync pass. On Spaces, pass an hf_token with "
138
+ "write access."
139
+ ),
140
+ structured_output=True,
141
+ )
142
+ def trigger_sync(
143
+ hf_token: str | None = None,
144
+ write_token: str | None = None,
145
+ ) -> bool:
146
+ _assert_mcp_mutation_access(hf_token=hf_token, write_token=write_token)
147
+ return trackio_server.force_sync()
148
+
149
+ mcp_app = mcp.streamable_http_app()
150
+
151
+ @asynccontextmanager
152
+ async def mcp_lifespan_context(app):
153
+ async with mcp.session_manager.run():
154
+ yield
155
+
156
+ return [Mount("/mcp", app=mcp_app)], mcp_lifespan_context
trackio/media/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Media module for Trackio.
3
+
4
+ This module contains all media-related functionality including:
5
+ - TrackioImage, TrackioVideo, TrackioAudio classes
6
+ - Video writing utilities
7
+ - Audio conversion utilities
8
+ """
9
+
10
+ from trackio.media.audio import TrackioAudio
11
+ from trackio.media.image import TrackioImage
12
+ from trackio.media.media import TrackioMedia
13
+ from trackio.media.utils import get_project_media_path
14
+ from trackio.media.video import TrackioVideo
15
+
16
+ write_audio = TrackioAudio.write_audio
17
+ write_video = TrackioVideo.write_video
18
+
19
+ __all__ = [
20
+ "TrackioMedia",
21
+ "TrackioImage",
22
+ "TrackioVideo",
23
+ "TrackioAudio",
24
+ "get_project_media_path",
25
+ "write_video",
26
+ "write_audio",
27
+ ]
trackio/media/audio.py ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import shutil
3
+ import warnings
4
+ from pathlib import Path
5
+ from typing import Literal
6
+
7
+ import numpy as np
8
+ from pydub import AudioSegment
9
+
10
+ from trackio.media.media import TrackioMedia
11
+ from trackio.media.utils import check_ffmpeg_installed, check_path
12
+
13
+ SUPPORTED_FORMATS = ["wav", "mp3"]
14
+ AudioFormatType = Literal["wav", "mp3"]
15
+ TrackioAudioSourceType = str | Path | np.ndarray
16
+
17
+
18
+ class TrackioAudio(TrackioMedia):
19
+ """
20
+ Initializes an Audio object.
21
+
22
+ Example:
23
+ ```python
24
+ import trackio
25
+ import numpy as np
26
+
27
+ # Generate a 1-second 440 Hz sine wave (mono)
28
+ sr = 16000
29
+ t = np.linspace(0, 1, sr, endpoint=False)
30
+ wave = 0.2 * np.sin(2 * np.pi * 440 * t)
31
+ audio = trackio.Audio(wave, caption="A4 sine", sample_rate=sr, format="wav")
32
+ trackio.log({"tone": audio})
33
+
34
+ # Stereo from numpy array (shape: samples, 2)
35
+ stereo = np.stack([wave, wave], axis=1)
36
+ audio = trackio.Audio(stereo, caption="Stereo", sample_rate=sr, format="mp3")
37
+ trackio.log({"stereo": audio})
38
+
39
+ # From an existing file
40
+ audio = trackio.Audio("path/to/audio.wav", caption="From file")
41
+ trackio.log({"file_audio": audio})
42
+ ```
43
+
44
+ Args:
45
+ value (`str`, `Path`, or `numpy.ndarray`, *optional*):
46
+ A path to an audio file, or a numpy array.
47
+ The array should be shaped `(samples,)` for mono or `(samples, 2)` for stereo.
48
+ Float arrays will be peak-normalized and converted to 16-bit PCM; integer arrays will be converted to 16-bit PCM as needed.
49
+ caption (`str`, *optional*):
50
+ A string caption for the audio.
51
+ sample_rate (`int`, *optional*):
52
+ Sample rate in Hz. Required when `value` is a numpy array.
53
+ format (`Literal["wav", "mp3"]`, *optional*):
54
+ Audio format used when `value` is a numpy array. Default is "wav".
55
+ """
56
+
57
+ TYPE = "trackio.audio"
58
+
59
+ def __init__(
60
+ self,
61
+ value: TrackioAudioSourceType,
62
+ caption: str | None = None,
63
+ sample_rate: int | None = None,
64
+ format: AudioFormatType | None = None,
65
+ ):
66
+ super().__init__(value, caption)
67
+ if isinstance(value, np.ndarray):
68
+ if sample_rate is None:
69
+ raise ValueError("Sample rate is required when value is an ndarray")
70
+ if format is None:
71
+ format = "wav"
72
+ self._format = format
73
+ self._sample_rate = sample_rate
74
+
75
+ def _save_media(self, file_path: Path):
76
+ if isinstance(self._value, np.ndarray):
77
+ TrackioAudio.write_audio(
78
+ data=self._value,
79
+ sample_rate=self._sample_rate,
80
+ filename=file_path,
81
+ format=self._format,
82
+ )
83
+ elif isinstance(self._value, str | Path):
84
+ if os.path.isfile(self._value):
85
+ shutil.copy(self._value, file_path)
86
+ else:
87
+ raise ValueError(f"File not found: {self._value}")
88
+
89
+ @staticmethod
90
+ def ensure_int16_pcm(data: np.ndarray) -> np.ndarray:
91
+ """
92
+ Convert input audio array to contiguous int16 PCM.
93
+ Peak normalization is applied to floating inputs.
94
+ """
95
+ arr = np.asarray(data)
96
+ if arr.ndim not in (1, 2):
97
+ raise ValueError("Audio data must be 1D (mono) or 2D ([samples, channels])")
98
+
99
+ if arr.dtype != np.int16:
100
+ warnings.warn(
101
+ f"Converting {arr.dtype} audio to int16 PCM; pass int16 to avoid conversion.",
102
+ stacklevel=2,
103
+ )
104
+
105
+ arr = np.nan_to_num(arr, copy=False)
106
+
107
+ # Floating types: normalize to peak 1.0, then scale to int16
108
+ if np.issubdtype(arr.dtype, np.floating):
109
+ max_abs = float(np.max(np.abs(arr))) if arr.size else 0.0
110
+ if max_abs > 0.0:
111
+ arr = arr / max_abs
112
+ out = (arr * 32767.0).clip(-32768, 32767).astype(np.int16, copy=False)
113
+ return np.ascontiguousarray(out)
114
+
115
+ converters: dict[np.dtype, callable] = {
116
+ np.dtype(np.int16): lambda a: a,
117
+ np.dtype(np.int32): lambda a: (a.astype(np.int32) // 65536).astype(
118
+ np.int16, copy=False
119
+ ),
120
+ np.dtype(np.uint16): lambda a: (a.astype(np.int32) - 32768).astype(
121
+ np.int16, copy=False
122
+ ),
123
+ np.dtype(np.uint8): lambda a: (a.astype(np.int32) * 257 - 32768).astype(
124
+ np.int16, copy=False
125
+ ),
126
+ np.dtype(np.int8): lambda a: (a.astype(np.int32) * 256).astype(
127
+ np.int16, copy=False
128
+ ),
129
+ }
130
+
131
+ conv = converters.get(arr.dtype)
132
+ if conv is not None:
133
+ out = conv(arr)
134
+ return np.ascontiguousarray(out)
135
+ raise TypeError(f"Unsupported audio dtype: {arr.dtype}")
136
+
137
+ @staticmethod
138
+ def write_audio(
139
+ data: np.ndarray,
140
+ sample_rate: int,
141
+ filename: str | Path,
142
+ format: AudioFormatType = "wav",
143
+ ) -> None:
144
+ if not isinstance(sample_rate, int) or sample_rate <= 0:
145
+ raise ValueError(f"Invalid sample_rate: {sample_rate}")
146
+ if format not in SUPPORTED_FORMATS:
147
+ raise ValueError(
148
+ f"Unsupported format: {format}. Supported: {SUPPORTED_FORMATS}"
149
+ )
150
+
151
+ check_path(filename)
152
+
153
+ pcm = TrackioAudio.ensure_int16_pcm(data)
154
+
155
+ if format != "wav":
156
+ check_ffmpeg_installed()
157
+
158
+ channels = 1 if pcm.ndim == 1 else pcm.shape[1]
159
+ audio = AudioSegment(
160
+ pcm.tobytes(),
161
+ frame_rate=sample_rate,
162
+ sample_width=2, # int16
163
+ channels=channels,
164
+ )
165
+
166
+ file = audio.export(str(filename), format=format)
167
+ file.close()
trackio/media/image.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import shutil
3
+ from pathlib import Path
4
+
5
+ import numpy as np
6
+ from PIL import Image as PILImage
7
+
8
+ from trackio.media.media import TrackioMedia
9
+
10
+ TrackioImageSourceType = str | Path | np.ndarray | PILImage.Image
11
+
12
+
13
+ class TrackioImage(TrackioMedia):
14
+ """
15
+ Initializes an Image object.
16
+
17
+ Example:
18
+ ```python
19
+ import trackio
20
+ import numpy as np
21
+ from PIL import Image
22
+
23
+ # Create an image from numpy array
24
+ image_data = np.random.randint(0, 255, (64, 64, 3), dtype=np.uint8)
25
+ image = trackio.Image(image_data, caption="Random image")
26
+ trackio.log({"my_image": image})
27
+
28
+ # Create an image from PIL Image
29
+ pil_image = Image.new('RGB', (100, 100), color='red')
30
+ image = trackio.Image(pil_image, caption="Red square")
31
+ trackio.log({"red_image": image})
32
+
33
+ # Create an image from file path
34
+ image = trackio.Image("path/to/image.jpg", caption="Photo from file")
35
+ trackio.log({"file_image": image})
36
+ ```
37
+
38
+ Args:
39
+ value (`str`, `Path`, `numpy.ndarray`, or `PIL.Image`, *optional*):
40
+ A path to an image, a PIL Image, or a numpy array of shape (height, width, channels).
41
+ If numpy array, should be of type `np.uint8` with RGB values in the range `[0, 255]`.
42
+ caption (`str`, *optional*):
43
+ A string caption for the image.
44
+ """
45
+
46
+ TYPE = "trackio.image"
47
+
48
+ def __init__(self, value: TrackioImageSourceType, caption: str | None = None):
49
+ super().__init__(value, caption)
50
+ self._format: str | None = None
51
+
52
+ if not isinstance(self._value, TrackioImageSourceType):
53
+ raise ValueError(
54
+ f"Invalid value type, expected {TrackioImageSourceType}, got {type(self._value)}"
55
+ )
56
+ if isinstance(self._value, np.ndarray) and self._value.dtype != np.uint8:
57
+ raise ValueError(
58
+ f"Invalid value dtype, expected np.uint8, got {self._value.dtype}"
59
+ )
60
+ if (
61
+ isinstance(self._value, np.ndarray | PILImage.Image)
62
+ and self._format is None
63
+ ):
64
+ self._format = "png"
65
+
66
+ def _as_pil(self) -> PILImage.Image | None:
67
+ try:
68
+ if isinstance(self._value, np.ndarray):
69
+ arr = np.asarray(self._value).astype("uint8")
70
+ return PILImage.fromarray(arr).convert("RGBA")
71
+ if isinstance(self._value, PILImage.Image):
72
+ return self._value.convert("RGBA")
73
+ except Exception as e:
74
+ raise ValueError(f"Failed to process image data: {self._value}") from e
75
+ return None
76
+
77
+ def _save_media(self, file_path: Path):
78
+ if pil := self._as_pil():
79
+ pil.save(file_path, format=self._format)
80
+ elif isinstance(self._value, str | Path):
81
+ if os.path.isfile(self._value):
82
+ shutil.copy(self._value, file_path)
83
+ else:
84
+ raise ValueError(f"File not found: {self._value}")
trackio/media/media.py ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import uuid
3
+ from abc import ABC, abstractmethod
4
+ from pathlib import Path
5
+
6
+ from trackio.media.utils import get_project_media_path
7
+ from trackio.utils import MEDIA_DIR
8
+
9
+
10
+ class TrackioMedia(ABC):
11
+ """
12
+ Abstract base class for Trackio media objects
13
+ Provides shared functionality for file handling and serialization.
14
+ """
15
+
16
+ TYPE: str
17
+
18
+ def __init_subclass__(cls, **kwargs):
19
+ """Ensure subclasses define the TYPE attribute."""
20
+ super().__init_subclass__(**kwargs)
21
+ if not hasattr(cls, "TYPE") or cls.TYPE is None:
22
+ raise TypeError(f"Class {cls.__name__} must define TYPE attribute")
23
+
24
+ def __init__(self, value, caption: str | None = None):
25
+ """
26
+ Saves the value and caption, and if the value is a file path, checks if the file exists.
27
+ """
28
+ self.caption = caption
29
+ self._value = value
30
+ self._file_path: Path | None = None
31
+
32
+ if isinstance(self._value, str | Path):
33
+ if not os.path.isfile(self._value):
34
+ raise ValueError(f"File not found: {self._value}")
35
+
36
+ def _file_extension(self) -> str:
37
+ if self._file_path:
38
+ return self._file_path.suffix[1:].lower()
39
+ if isinstance(self._value, str | Path):
40
+ path = Path(self._value)
41
+ return path.suffix[1:].lower()
42
+ if hasattr(self, "_format") and self._format:
43
+ return self._format
44
+ return "unknown"
45
+
46
+ def _get_relative_file_path(self) -> Path | None:
47
+ return self._file_path
48
+
49
+ def _get_absolute_file_path(self) -> Path | None:
50
+ if self._file_path:
51
+ return MEDIA_DIR / self._file_path
52
+ return None
53
+
54
+ def _save(self, project: str, run: str, step: int = 0):
55
+ if self._file_path:
56
+ return
57
+
58
+ media_dir = get_project_media_path(project=project, run=run, step=step)
59
+ filename = f"{uuid.uuid4()}.{self._file_extension()}"
60
+ file_path = media_dir / filename
61
+
62
+ self._save_media(file_path)
63
+ self._file_path = file_path.relative_to(MEDIA_DIR)
64
+
65
+ @abstractmethod
66
+ def _save_media(self, file_path: Path):
67
+ """
68
+ Performs the actual media saving logic.
69
+ """
70
+ pass
71
+
72
+ def _to_dict(self) -> dict:
73
+ if not self._file_path:
74
+ raise ValueError("Media must be saved to file before serialization")
75
+ return {
76
+ "_type": self.TYPE,
77
+ "file_path": str(self._get_relative_file_path()),
78
+ "caption": self.caption,
79
+ }
trackio/media/utils.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import shutil
2
+ from pathlib import Path
3
+
4
+ from trackio.utils import MEDIA_DIR
5
+
6
+
7
+ def check_path(file_path: str | Path) -> None:
8
+ """Raise an error if the parent directory does not exist."""
9
+ file_path = Path(file_path)
10
+ if not file_path.parent.exists():
11
+ try:
12
+ file_path.parent.mkdir(parents=True, exist_ok=True)
13
+ except OSError as e:
14
+ raise ValueError(
15
+ f"Failed to create parent directory {file_path.parent}: {e}"
16
+ )
17
+
18
+
19
+ def check_ffmpeg_installed() -> None:
20
+ """Raise an error if ffmpeg is not available on the system PATH."""
21
+ if shutil.which("ffmpeg") is None:
22
+ raise RuntimeError(
23
+ "ffmpeg is required to write video but was not found on your system. "
24
+ "Please install ffmpeg and ensure it is available on your PATH."
25
+ )
26
+
27
+
28
+ def get_project_media_path(
29
+ project: str,
30
+ run: str | None = None,
31
+ step: int | None = None,
32
+ relative_path: str | Path | None = None,
33
+ ) -> Path:
34
+ """
35
+ Get the full path where uploaded files are stored for a Trackio project (and create the directory if it doesn't exist).
36
+ If a run is not provided, the files are stored in a project-level directory with the given relative path.
37
+
38
+ Args:
39
+ project: The project name
40
+ run: The run name
41
+ step: The step number
42
+ relative_path: The relative path within the directory (only used if run is not provided)
43
+
44
+ Returns:
45
+ The full path to the media file
46
+ """
47
+ if step is not None and run is None:
48
+ raise ValueError("Uploading files at a specific step requires a run")
49
+
50
+ path = MEDIA_DIR / project
51
+ if run:
52
+ path /= run
53
+ if step is not None:
54
+ path /= str(step)
55
+ else:
56
+ path /= "files"
57
+ if relative_path:
58
+ path /= relative_path
59
+ path.mkdir(parents=True, exist_ok=True)
60
+ return path
trackio/media/video.py ADDED
@@ -0,0 +1,246 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import shutil
3
+ import subprocess
4
+ from pathlib import Path
5
+ from typing import Literal
6
+
7
+ import numpy as np
8
+
9
+ from trackio.media.media import TrackioMedia
10
+ from trackio.media.utils import check_ffmpeg_installed, check_path
11
+
12
+ TrackioVideoSourceType = str | Path | np.ndarray
13
+ TrackioVideoFormatType = Literal["gif", "mp4", "webm"]
14
+ VideoCodec = Literal["h264", "vp9", "gif"]
15
+
16
+
17
+ class TrackioVideo(TrackioMedia):
18
+ """
19
+ Initializes a Video object.
20
+
21
+ Example:
22
+ ```python
23
+ import trackio
24
+ import numpy as np
25
+
26
+ # Create a simple video from numpy array
27
+ frames = np.random.randint(0, 255, (10, 3, 64, 64), dtype=np.uint8)
28
+ video = trackio.Video(frames, caption="Random video", fps=30)
29
+
30
+ # Create a batch of videos
31
+ batch_frames = np.random.randint(0, 255, (3, 10, 3, 64, 64), dtype=np.uint8)
32
+ batch_video = trackio.Video(batch_frames, caption="Batch of videos", fps=15)
33
+
34
+ # Create video from file path
35
+ video = trackio.Video("path/to/video.mp4", caption="Video from file")
36
+ ```
37
+
38
+ Args:
39
+ value (`str`, `Path`, or `numpy.ndarray`, *optional*):
40
+ A path to a video file, or a numpy array.
41
+ If numpy array, should be of type `np.uint8` with RGB values in the range `[0, 255]`.
42
+ It is expected to have shape of either (frames, channels, height, width) or (batch, frames, channels, height, width).
43
+ For the latter, the videos will be tiled into a grid.
44
+ caption (`str`, *optional*):
45
+ A string caption for the video.
46
+ fps (`int`, *optional*):
47
+ Frames per second for the video. Only used when value is an ndarray. Default is `24`.
48
+ format (`Literal["gif", "mp4", "webm"]`, *optional*):
49
+ Video format ("gif", "mp4", or "webm"). Only used when value is an ndarray. Default is "gif".
50
+ """
51
+
52
+ TYPE = "trackio.video"
53
+
54
+ def __init__(
55
+ self,
56
+ value: TrackioVideoSourceType,
57
+ caption: str | None = None,
58
+ fps: int | None = None,
59
+ format: TrackioVideoFormatType | None = None,
60
+ ):
61
+ super().__init__(value, caption)
62
+
63
+ if not isinstance(self._value, TrackioVideoSourceType):
64
+ raise ValueError(
65
+ f"Invalid value type, expected {TrackioVideoSourceType}, got {type(self._value)}"
66
+ )
67
+ if isinstance(self._value, np.ndarray):
68
+ if self._value.dtype != np.uint8:
69
+ raise ValueError(
70
+ f"Invalid value dtype, expected np.uint8, got {self._value.dtype}"
71
+ )
72
+ if format is None:
73
+ format = "gif"
74
+ if fps is None:
75
+ fps = 24
76
+ self._fps = fps
77
+ self._format = format
78
+
79
+ @staticmethod
80
+ def _check_array_format(video: np.ndarray) -> None:
81
+ """Raise an error if the array is not in the expected format."""
82
+ if not (video.ndim == 4 and video.shape[-1] == 3):
83
+ raise ValueError(
84
+ f"Expected RGB input shaped (F, H, W, 3), got {video.shape}. "
85
+ f"Input has {video.ndim} dimensions, expected 4."
86
+ )
87
+ if video.dtype != np.uint8:
88
+ raise TypeError(
89
+ f"Expected dtype=uint8, got {video.dtype}. "
90
+ "Please convert your video data to uint8 format."
91
+ )
92
+
93
+ @staticmethod
94
+ def write_video(
95
+ file_path: str | Path, video: np.ndarray, fps: float, codec: VideoCodec
96
+ ) -> None:
97
+ """RGB uint8 only, shape (F, H, W, 3)."""
98
+ check_ffmpeg_installed()
99
+ check_path(file_path)
100
+
101
+ if codec not in {"h264", "vp9", "gif"}:
102
+ raise ValueError("Unsupported codec. Use h264, vp9, or gif.")
103
+
104
+ arr = np.asarray(video)
105
+ TrackioVideo._check_array_format(arr)
106
+
107
+ frames = np.ascontiguousarray(arr)
108
+ _, height, width, _ = frames.shape
109
+ out_path = str(file_path)
110
+
111
+ cmd = [
112
+ "ffmpeg",
113
+ "-y",
114
+ "-f",
115
+ "rawvideo",
116
+ "-s",
117
+ f"{width}x{height}",
118
+ "-pix_fmt",
119
+ "rgb24",
120
+ "-r",
121
+ str(fps),
122
+ "-i",
123
+ "-",
124
+ "-an",
125
+ ]
126
+
127
+ if codec == "gif":
128
+ video_filter = "split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse"
129
+ cmd += [
130
+ "-vf",
131
+ video_filter,
132
+ "-loop",
133
+ "0",
134
+ ]
135
+ elif codec == "h264":
136
+ cmd += [
137
+ "-vcodec",
138
+ "libx264",
139
+ "-pix_fmt",
140
+ "yuv420p",
141
+ "-movflags",
142
+ "+faststart",
143
+ ]
144
+ elif codec == "vp9":
145
+ bpp = 0.08
146
+ bps = int(width * height * fps * bpp)
147
+ if bps >= 1_000_000:
148
+ bitrate = f"{round(bps / 1_000_000)}M"
149
+ elif bps >= 1_000:
150
+ bitrate = f"{round(bps / 1_000)}k"
151
+ else:
152
+ bitrate = str(max(bps, 1))
153
+ cmd += [
154
+ "-vcodec",
155
+ "libvpx-vp9",
156
+ "-b:v",
157
+ bitrate,
158
+ "-pix_fmt",
159
+ "yuv420p",
160
+ ]
161
+ cmd += [out_path]
162
+ proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
163
+ try:
164
+ for frame in frames:
165
+ proc.stdin.write(frame.tobytes())
166
+ finally:
167
+ if proc.stdin:
168
+ proc.stdin.close()
169
+ stderr = (
170
+ proc.stderr.read().decode("utf-8", errors="ignore")
171
+ if proc.stderr
172
+ else ""
173
+ )
174
+ ret = proc.wait()
175
+ if ret != 0:
176
+ raise RuntimeError(f"ffmpeg failed with code {ret}\n{stderr}")
177
+
178
+ @property
179
+ def _codec(self) -> str:
180
+ match self._format:
181
+ case "gif":
182
+ return "gif"
183
+ case "mp4":
184
+ return "h264"
185
+ case "webm":
186
+ return "vp9"
187
+ case _:
188
+ raise ValueError(f"Unsupported format: {self._format}")
189
+
190
+ def _save_media(self, file_path: Path):
191
+ if isinstance(self._value, np.ndarray):
192
+ video = TrackioVideo._process_ndarray(self._value)
193
+ TrackioVideo.write_video(file_path, video, fps=self._fps, codec=self._codec)
194
+ elif isinstance(self._value, str | Path):
195
+ if os.path.isfile(self._value):
196
+ shutil.copy(self._value, file_path)
197
+ else:
198
+ raise ValueError(f"File not found: {self._value}")
199
+
200
+ @staticmethod
201
+ def _process_ndarray(value: np.ndarray) -> np.ndarray:
202
+ # Verify value is either 4D (single video) or 5D array (batched videos).
203
+ # Expected format: (frames, channels, height, width) or (batch, frames, channels, height, width)
204
+ if value.ndim < 4:
205
+ raise ValueError(
206
+ "Video requires at least 4 dimensions (frames, channels, height, width)"
207
+ )
208
+ if value.ndim > 5:
209
+ raise ValueError(
210
+ "Videos can have at most 5 dimensions (batch, frames, channels, height, width)"
211
+ )
212
+ if value.ndim == 4:
213
+ # Reshape to 5D with single batch: (1, frames, channels, height, width)
214
+ value = value[np.newaxis, ...]
215
+
216
+ value = TrackioVideo._tile_batched_videos(value)
217
+ return value
218
+
219
+ @staticmethod
220
+ def _tile_batched_videos(video: np.ndarray) -> np.ndarray:
221
+ """
222
+ Tiles a batch of videos into a grid of videos.
223
+
224
+ Input format: (batch, frames, channels, height, width) - original FCHW format
225
+ Output format: (frames, total_height, total_width, channels)
226
+ """
227
+ batch_size, frames, channels, height, width = video.shape
228
+
229
+ next_pow2 = 1 << (batch_size - 1).bit_length()
230
+ if batch_size != next_pow2:
231
+ pad_len = next_pow2 - batch_size
232
+ pad_shape = (pad_len, frames, channels, height, width)
233
+ padding = np.zeros(pad_shape, dtype=video.dtype)
234
+ video = np.concatenate((video, padding), axis=0)
235
+ batch_size = next_pow2
236
+
237
+ n_rows = 1 << ((batch_size.bit_length() - 1) // 2)
238
+ n_cols = batch_size // n_rows
239
+
240
+ # Reshape to grid layout: (n_rows, n_cols, frames, channels, height, width)
241
+ video = video.reshape(n_rows, n_cols, frames, channels, height, width)
242
+
243
+ # Rearrange dimensions to (frames, total_height, total_width, channels)
244
+ video = video.transpose(2, 0, 4, 1, 5, 3)
245
+ video = video.reshape(frames, n_rows * height, n_cols * width, channels)
246
+ return video
trackio/package.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "name": "trackio",
3
+ "version": "0.22.0",
4
+ "description": "",
5
+ "python": "true"
6
+ }
trackio/py.typed ADDED
File without changes
trackio/remote_client.py ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+ from typing import Any
5
+ from urllib.parse import urljoin
6
+
7
+ import httpx
8
+ from gradio_client import Client as GradioClient
9
+ from huggingface_hub.utils import build_hf_headers
10
+
11
+ HTTP_API_VERSION = 1
12
+
13
+
14
+ def _normalize_src(src: str) -> str:
15
+ return src if src.endswith("/") else src + "/"
16
+
17
+
18
+ def _space_id_to_url(space_id: str) -> str:
19
+ namespace, name = space_id.split("/", 1)
20
+ return f"https://{namespace}-{name}.hf.space/"
21
+
22
+
23
+ def _resolve_src_url(src: str) -> str:
24
+ if src.startswith(("http://", "https://")):
25
+ return _normalize_src(src)
26
+ if "/" in src:
27
+ return _space_id_to_url(src)
28
+ raise ValueError(
29
+ f"Could not resolve Trackio remote source '{src}'. "
30
+ "Pass a full Space id like 'user/space' or a URL."
31
+ )
32
+
33
+
34
+ def _is_local_file_data(value: Any) -> bool:
35
+ return (
36
+ isinstance(value, dict)
37
+ and "path" in value
38
+ and isinstance(value["path"], str)
39
+ and value.get("meta", {}).get("_type") == "gradio.FileData"
40
+ and Path(value["path"]).exists()
41
+ )
42
+
43
+
44
+ class _TrackioHTTPClient:
45
+ def __init__(
46
+ self,
47
+ src: str,
48
+ hf_token: str | None = None,
49
+ httpx_kwargs: dict[str, Any] | None = None,
50
+ ) -> None:
51
+ self.src = _resolve_src_url(src)
52
+ self.httpx_kwargs = dict(httpx_kwargs or {})
53
+ self.headers = build_hf_headers(token=hf_token)
54
+
55
+ def _upload_file(self, file_data: dict[str, Any]) -> dict[str, Any]:
56
+ path = Path(file_data["path"])
57
+ with path.open("rb") as f:
58
+ resp = httpx.post(
59
+ urljoin(self.src, "api/upload"),
60
+ headers=self.headers,
61
+ files={"files": (path.name, f)},
62
+ **self.httpx_kwargs,
63
+ )
64
+ resp.raise_for_status()
65
+ uploaded_path = resp.json()["paths"][0]
66
+ return {
67
+ **file_data,
68
+ "path": uploaded_path,
69
+ "orig_name": file_data.get("orig_name", path.name),
70
+ }
71
+
72
+ def _prepare_value(self, value: Any) -> Any:
73
+ if _is_local_file_data(value):
74
+ return self._upload_file(value)
75
+ if isinstance(value, list):
76
+ return [self._prepare_value(item) for item in value]
77
+ if isinstance(value, tuple):
78
+ return [self._prepare_value(item) for item in value]
79
+ if isinstance(value, dict):
80
+ return {k: self._prepare_value(v) for k, v in value.items()}
81
+ return value
82
+
83
+ def predict(self, *args, api_name: str, **kwargs) -> Any:
84
+ api_name = api_name.lstrip("/")
85
+ payload = {
86
+ "args": self._prepare_value(list(args)),
87
+ "kwargs": self._prepare_value(kwargs),
88
+ }
89
+ resp = httpx.post(
90
+ urljoin(self.src, f"api/{api_name}"),
91
+ headers=self.headers,
92
+ json=payload,
93
+ **self.httpx_kwargs,
94
+ )
95
+ if resp.status_code == 404:
96
+ raise RuntimeError(
97
+ f"Space '{self.src}' does not support '/{api_name}'. Redeploy with `trackio sync`."
98
+ )
99
+ resp.raise_for_status()
100
+ body = resp.json()
101
+ if body.get("error") is not None:
102
+ raise RuntimeError(body["error"])
103
+ return body.get("data")
104
+
105
+
106
+ class _TrackioGradioCompatClient:
107
+ def __init__(
108
+ self,
109
+ src: str,
110
+ hf_token: str | None = None,
111
+ httpx_kwargs: dict[str, Any] | None = None,
112
+ verbose: bool = False,
113
+ ) -> None:
114
+ kwargs: dict[str, Any] = {"verbose": verbose}
115
+ if hf_token:
116
+ kwargs["hf_token"] = hf_token
117
+ if httpx_kwargs:
118
+ kwargs["httpx_kwargs"] = httpx_kwargs
119
+ self._client = GradioClient(src, **kwargs)
120
+
121
+ def predict(self, *args, api_name: str, **kwargs) -> Any:
122
+ try:
123
+ return self._client.predict(*args, api_name=api_name, **kwargs)
124
+ except Exception as e:
125
+ if "API Not Found" in str(e) or "api_name" in str(e):
126
+ raise RuntimeError(
127
+ f"Space '{self._client.src}' does not support '{api_name}'. "
128
+ "Redeploy with `trackio sync`."
129
+ ) from e
130
+ raise
131
+
132
+
133
+ def _supports_http_api(
134
+ src: str,
135
+ hf_token: str | None = None,
136
+ httpx_kwargs: dict[str, Any] | None = None,
137
+ ) -> bool:
138
+ url = _resolve_src_url(src)
139
+ headers = build_hf_headers(token=hf_token)
140
+ kwargs = dict(httpx_kwargs or {})
141
+ kwargs.setdefault("timeout", 10)
142
+ try:
143
+ resp = httpx.get(urljoin(url, "version"), headers=headers, **kwargs)
144
+ if not resp.is_success:
145
+ return False
146
+ data = resp.json()
147
+ return data.get("api_version") == HTTP_API_VERSION
148
+ except Exception:
149
+ return False
150
+
151
+
152
+ class RemoteClient:
153
+ def __init__(
154
+ self,
155
+ space: str,
156
+ hf_token: str | None = None,
157
+ httpx_kwargs: dict[str, Any] | None = None,
158
+ verbose: bool = False,
159
+ ) -> None:
160
+ self._space = space
161
+ try:
162
+ if _supports_http_api(space, hf_token=hf_token, httpx_kwargs=httpx_kwargs):
163
+ self._client = _TrackioHTTPClient(
164
+ space, hf_token=hf_token, httpx_kwargs=httpx_kwargs
165
+ )
166
+ else:
167
+ self._client = _TrackioGradioCompatClient(
168
+ space,
169
+ hf_token=hf_token,
170
+ httpx_kwargs=httpx_kwargs,
171
+ verbose=verbose,
172
+ )
173
+ except ValueError:
174
+ raise
175
+ except Exception as e:
176
+ raise ConnectionError(
177
+ f"Could not connect to Space '{space}'. Is it running?\n{e}"
178
+ ) from e
179
+
180
+ def predict(self, *args, api_name: str, **kwargs) -> Any:
181
+ return self._client.predict(*args, api_name=api_name, **kwargs)
trackio/run.py ADDED
@@ -0,0 +1,745 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import shutil
3
+ import threading
4
+ import uuid
5
+ import warnings
6
+ from datetime import datetime, timezone
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ import huggingface_hub
11
+ from gradio_client import handle_file
12
+
13
+ from trackio import utils
14
+ from trackio.alerts import (
15
+ AlertLevel,
16
+ format_alert_terminal,
17
+ resolve_webhook_min_level,
18
+ send_webhook,
19
+ should_send_webhook,
20
+ )
21
+ from trackio.apple_gpu import AppleGpuMonitor, apple_gpu_available
22
+ from trackio.gpu import GpuMonitor, gpu_available
23
+ from trackio.histogram import Histogram
24
+ from trackio.markdown import Markdown
25
+ from trackio.media import TrackioMedia, get_project_media_path
26
+ from trackio.remote_client import RemoteClient
27
+ from trackio.sqlite_storage import SQLiteStorage
28
+ from trackio.table import Table
29
+ from trackio.typehints import AlertEntry, LogEntry, SystemLogEntry, UploadEntry
30
+ from trackio.utils import _get_default_namespace
31
+
32
+ BATCH_SEND_INTERVAL = 0.5
33
+ MAX_BACKOFF = 30
34
+
35
+
36
+ class Run:
37
+ def __init__(
38
+ self,
39
+ url: str | None,
40
+ project: str,
41
+ client: Any | None,
42
+ name: str | None = None,
43
+ group: str | None = None,
44
+ config: dict | None = None,
45
+ space_id: str | None = None,
46
+ auto_log_gpu: bool = False,
47
+ gpu_log_interval: float = 10.0,
48
+ webhook_url: str | None = None,
49
+ webhook_min_level: AlertLevel | str | None = None,
50
+ ):
51
+ """
52
+ Initialize a Run for logging metrics to Trackio.
53
+
54
+ Args:
55
+ url: The URL or Space id of the Trackio server.
56
+ project: The name of the project to log metrics to.
57
+ client: A pre-configured Trackio-compatible client instance, or None to
58
+ create one automatically in a background thread with retry logic.
59
+ Passing None is recommended for normal usage. Passing a client
60
+ is useful for testing (e.g., injecting a mock client).
61
+ name: The name of this run. If None, a readable name like
62
+ "brave-sunset-0" is auto-generated. If space_id is provided,
63
+ generates a "username-timestamp" format instead.
64
+ group: Optional group name to organize related runs together.
65
+ config: A dictionary of configuration/hyperparameters for this run.
66
+ Keys starting with '_' are reserved for internal use.
67
+ space_id: The HF Space ID if logging to a Space (e.g., "user/space").
68
+ If provided, media files will be uploaded to the Space.
69
+ auto_log_gpu: Whether to automatically log GPU metrics (utilization,
70
+ memory, temperature) at regular intervals.
71
+ gpu_log_interval: The interval in seconds between GPU metric logs.
72
+ Only used when auto_log_gpu is True.
73
+ webhook_url: A webhook URL to POST alert payloads to. Supports
74
+ Slack and Discord webhook URLs natively. Can also be set via
75
+ the TRACKIO_WEBHOOK_URL environment variable.
76
+ webhook_min_level: Minimum alert level that should trigger webhook
77
+ delivery. For example, `AlertLevel.WARN` sends only WARN and
78
+ ERROR alerts to webhook destinations. Can also be set via
79
+ `TRACKIO_WEBHOOK_MIN_LEVEL`.
80
+ """
81
+ self.url = url
82
+ self.project = project
83
+ self._client_lock = threading.Lock()
84
+ self._client_thread = None
85
+ self._client = client
86
+ self._space_id = space_id
87
+ self.name = name or utils.generate_readable_name(
88
+ SQLiteStorage.get_runs(project), space_id
89
+ )
90
+ self.group = group
91
+ self.config = utils.to_json_safe(config or {})
92
+
93
+ if isinstance(self.config, dict):
94
+ for key in self.config:
95
+ if key.startswith("_"):
96
+ raise ValueError(
97
+ f"Config key '{key}' is reserved (keys starting with '_' are reserved for internal use)"
98
+ )
99
+
100
+ self.config["_Username"] = self._get_username()
101
+ self.config["_Created"] = datetime.now(timezone.utc).isoformat()
102
+ self.config["_Group"] = self.group
103
+
104
+ self._queued_logs: list[LogEntry] = []
105
+ self._queued_system_logs: list[SystemLogEntry] = []
106
+ self._queued_uploads: list[UploadEntry] = []
107
+ self._queued_alerts: list[AlertEntry] = []
108
+ self._stop_flag = threading.Event()
109
+ self._config_logged = False
110
+ max_step = SQLiteStorage.get_max_step_for_run(self.project, self.name)
111
+ self._next_step = 0 if max_step is None else max_step + 1
112
+ self._has_local_buffer = False
113
+
114
+ self._is_local = space_id is None
115
+ self._webhook_url = webhook_url or os.environ.get("TRACKIO_WEBHOOK_URL")
116
+ self._webhook_min_level = resolve_webhook_min_level(
117
+ webhook_min_level or os.environ.get("TRACKIO_WEBHOOK_MIN_LEVEL")
118
+ )
119
+
120
+ if self._is_local:
121
+ self._local_sender_thread = threading.Thread(
122
+ target=self._local_batch_sender
123
+ )
124
+ self._local_sender_thread.daemon = True
125
+ self._local_sender_thread.start()
126
+ else:
127
+ self._client_thread = threading.Thread(target=self._init_client_background)
128
+ self._client_thread.daemon = True
129
+ self._client_thread.start()
130
+
131
+ self._gpu_monitor: "GpuMonitor | AppleGpuMonitor | None" = None
132
+ if auto_log_gpu:
133
+ if gpu_available():
134
+ self._gpu_monitor = GpuMonitor(self, interval=gpu_log_interval)
135
+ self._gpu_monitor.start()
136
+ elif apple_gpu_available():
137
+ self._gpu_monitor = AppleGpuMonitor(self, interval=gpu_log_interval)
138
+ self._gpu_monitor.start()
139
+
140
+ def _get_username(self) -> str | None:
141
+ try:
142
+ return _get_default_namespace()
143
+ except Exception:
144
+ return None
145
+
146
+ def _local_batch_sender(self):
147
+ while (
148
+ not self._stop_flag.is_set()
149
+ or len(self._queued_logs) > 0
150
+ or len(self._queued_system_logs) > 0
151
+ or len(self._queued_alerts) > 0
152
+ ):
153
+ if not self._stop_flag.is_set():
154
+ self._stop_flag.wait(timeout=BATCH_SEND_INTERVAL)
155
+
156
+ with self._client_lock:
157
+ if self._queued_logs:
158
+ logs_to_send = self._queued_logs.copy()
159
+ self._queued_logs.clear()
160
+ self._write_logs_to_sqlite(logs_to_send)
161
+
162
+ if self._queued_system_logs:
163
+ system_logs_to_send = self._queued_system_logs.copy()
164
+ self._queued_system_logs.clear()
165
+ self._write_system_logs_to_sqlite(system_logs_to_send)
166
+
167
+ if self._queued_alerts:
168
+ alerts_to_send = self._queued_alerts.copy()
169
+ self._queued_alerts.clear()
170
+ self._write_alerts_to_sqlite(alerts_to_send)
171
+
172
+ def _write_logs_to_sqlite(self, logs: list[LogEntry]):
173
+ logs_by_run: dict[tuple, dict] = {}
174
+ for entry in logs:
175
+ key = (entry["project"], entry["run"])
176
+ if key not in logs_by_run:
177
+ logs_by_run[key] = {
178
+ "metrics": [],
179
+ "steps": [],
180
+ "log_ids": [],
181
+ "config": None,
182
+ }
183
+ logs_by_run[key]["metrics"].append(entry["metrics"])
184
+ logs_by_run[key]["steps"].append(entry.get("step"))
185
+ logs_by_run[key]["log_ids"].append(entry.get("log_id"))
186
+ if entry.get("config") and logs_by_run[key]["config"] is None:
187
+ logs_by_run[key]["config"] = entry["config"]
188
+
189
+ for (project, run), data in logs_by_run.items():
190
+ has_log_ids = any(lid is not None for lid in data["log_ids"])
191
+ SQLiteStorage.bulk_log(
192
+ project=project,
193
+ run=run,
194
+ metrics_list=data["metrics"],
195
+ steps=data["steps"],
196
+ config=data["config"],
197
+ log_ids=data["log_ids"] if has_log_ids else None,
198
+ )
199
+
200
+ def _write_system_logs_to_sqlite(self, logs: list[SystemLogEntry]):
201
+ logs_by_run: dict[tuple, dict] = {}
202
+ for entry in logs:
203
+ key = (entry["project"], entry["run"])
204
+ if key not in logs_by_run:
205
+ logs_by_run[key] = {"metrics": [], "timestamps": [], "log_ids": []}
206
+ logs_by_run[key]["metrics"].append(entry["metrics"])
207
+ logs_by_run[key]["timestamps"].append(entry.get("timestamp"))
208
+ logs_by_run[key]["log_ids"].append(entry.get("log_id"))
209
+
210
+ for (project, run), data in logs_by_run.items():
211
+ has_log_ids = any(lid is not None for lid in data["log_ids"])
212
+ SQLiteStorage.bulk_log_system(
213
+ project=project,
214
+ run=run,
215
+ metrics_list=data["metrics"],
216
+ timestamps=data["timestamps"],
217
+ log_ids=data["log_ids"] if has_log_ids else None,
218
+ )
219
+
220
+ def _write_alerts_to_sqlite(self, alerts: list[AlertEntry]):
221
+ alerts_by_run: dict[tuple, dict] = {}
222
+ for entry in alerts:
223
+ key = (entry["project"], entry["run"])
224
+ if key not in alerts_by_run:
225
+ alerts_by_run[key] = {
226
+ "titles": [],
227
+ "texts": [],
228
+ "levels": [],
229
+ "steps": [],
230
+ "timestamps": [],
231
+ "alert_ids": [],
232
+ }
233
+ alerts_by_run[key]["titles"].append(entry["title"])
234
+ alerts_by_run[key]["texts"].append(entry.get("text"))
235
+ alerts_by_run[key]["levels"].append(entry["level"])
236
+ alerts_by_run[key]["steps"].append(entry.get("step"))
237
+ alerts_by_run[key]["timestamps"].append(entry.get("timestamp"))
238
+ alerts_by_run[key]["alert_ids"].append(entry.get("alert_id"))
239
+
240
+ for (project, run), data in alerts_by_run.items():
241
+ has_alert_ids = any(aid is not None for aid in data["alert_ids"])
242
+ SQLiteStorage.bulk_alert(
243
+ project=project,
244
+ run=run,
245
+ titles=data["titles"],
246
+ texts=data["texts"],
247
+ levels=data["levels"],
248
+ steps=data["steps"],
249
+ timestamps=data["timestamps"],
250
+ alert_ids=data["alert_ids"] if has_alert_ids else None,
251
+ )
252
+
253
+ def _batch_sender(self):
254
+ consecutive_failures = 0
255
+ while (
256
+ not self._stop_flag.is_set()
257
+ or len(self._queued_logs) > 0
258
+ or len(self._queued_system_logs) > 0
259
+ or len(self._queued_uploads) > 0
260
+ or len(self._queued_alerts) > 0
261
+ or self._has_local_buffer
262
+ ):
263
+ if not self._stop_flag.is_set():
264
+ if consecutive_failures:
265
+ sleep_time = min(
266
+ BATCH_SEND_INTERVAL * (2**consecutive_failures), MAX_BACKOFF
267
+ )
268
+ else:
269
+ sleep_time = BATCH_SEND_INTERVAL
270
+ self._stop_flag.wait(timeout=sleep_time)
271
+ elif self._has_local_buffer:
272
+ self._stop_flag.wait(timeout=BATCH_SEND_INTERVAL)
273
+
274
+ with self._client_lock:
275
+ if self._client is None:
276
+ if self._stop_flag.is_set():
277
+ if self._queued_logs:
278
+ self._persist_logs_locally(self._queued_logs)
279
+ self._queued_logs.clear()
280
+ if self._queued_system_logs:
281
+ self._persist_system_logs_locally(self._queued_system_logs)
282
+ self._queued_system_logs.clear()
283
+ if self._queued_uploads:
284
+ self._persist_uploads_locally(self._queued_uploads)
285
+ self._queued_uploads.clear()
286
+ if self._queued_alerts:
287
+ self._write_alerts_to_sqlite(self._queued_alerts)
288
+ self._queued_alerts.clear()
289
+ return
290
+
291
+ failed = False
292
+
293
+ if self._queued_logs:
294
+ logs_to_send = self._queued_logs.copy()
295
+ self._queued_logs.clear()
296
+ try:
297
+ self._client.predict(
298
+ api_name="/bulk_log",
299
+ logs=logs_to_send,
300
+ hf_token=huggingface_hub.utils.get_token(),
301
+ )
302
+ except Exception:
303
+ self._persist_logs_locally(logs_to_send)
304
+ failed = True
305
+
306
+ if self._queued_system_logs:
307
+ system_logs_to_send = self._queued_system_logs.copy()
308
+ self._queued_system_logs.clear()
309
+ try:
310
+ self._client.predict(
311
+ api_name="/bulk_log_system",
312
+ logs=system_logs_to_send,
313
+ hf_token=huggingface_hub.utils.get_token(),
314
+ )
315
+ except Exception:
316
+ self._persist_system_logs_locally(system_logs_to_send)
317
+ failed = True
318
+
319
+ if self._queued_uploads:
320
+ uploads_to_send = self._queued_uploads.copy()
321
+ self._queued_uploads.clear()
322
+ try:
323
+ self._client.predict(
324
+ api_name="/bulk_upload_media",
325
+ uploads=uploads_to_send,
326
+ hf_token=huggingface_hub.utils.get_token(),
327
+ )
328
+ except Exception:
329
+ self._persist_uploads_locally(uploads_to_send)
330
+ failed = True
331
+
332
+ if self._queued_alerts:
333
+ alerts_to_send = self._queued_alerts.copy()
334
+ self._queued_alerts.clear()
335
+ try:
336
+ self._client.predict(
337
+ api_name="/bulk_alert",
338
+ alerts=alerts_to_send,
339
+ hf_token=huggingface_hub.utils.get_token(),
340
+ )
341
+ except Exception:
342
+ self._write_alerts_to_sqlite(alerts_to_send)
343
+ failed = True
344
+
345
+ if failed:
346
+ consecutive_failures += 1
347
+ else:
348
+ consecutive_failures = 0
349
+ if self._has_local_buffer:
350
+ self._flush_local_buffer()
351
+
352
+ def _persist_logs_locally(self, logs: list[LogEntry]):
353
+ if not self._space_id:
354
+ return
355
+ logs_by_run: dict[tuple, dict] = {}
356
+ for entry in logs:
357
+ key = (entry["project"], entry["run"])
358
+ if key not in logs_by_run:
359
+ logs_by_run[key] = {
360
+ "metrics": [],
361
+ "steps": [],
362
+ "log_ids": [],
363
+ "config": None,
364
+ }
365
+ logs_by_run[key]["metrics"].append(entry["metrics"])
366
+ logs_by_run[key]["steps"].append(entry.get("step"))
367
+ logs_by_run[key]["log_ids"].append(entry.get("log_id"))
368
+ if entry.get("config") and logs_by_run[key]["config"] is None:
369
+ logs_by_run[key]["config"] = entry["config"]
370
+
371
+ for (project, run), data in logs_by_run.items():
372
+ SQLiteStorage.bulk_log(
373
+ project=project,
374
+ run=run,
375
+ metrics_list=data["metrics"],
376
+ steps=data["steps"],
377
+ log_ids=data["log_ids"],
378
+ config=data["config"],
379
+ space_id=self._space_id,
380
+ )
381
+ self._has_local_buffer = True
382
+
383
+ def _persist_system_logs_locally(self, logs: list[SystemLogEntry]):
384
+ if not self._space_id:
385
+ return
386
+ logs_by_run: dict[tuple, dict] = {}
387
+ for entry in logs:
388
+ key = (entry["project"], entry["run"])
389
+ if key not in logs_by_run:
390
+ logs_by_run[key] = {"metrics": [], "timestamps": [], "log_ids": []}
391
+ logs_by_run[key]["metrics"].append(entry["metrics"])
392
+ logs_by_run[key]["timestamps"].append(entry.get("timestamp"))
393
+ logs_by_run[key]["log_ids"].append(entry.get("log_id"))
394
+
395
+ for (project, run), data in logs_by_run.items():
396
+ SQLiteStorage.bulk_log_system(
397
+ project=project,
398
+ run=run,
399
+ metrics_list=data["metrics"],
400
+ timestamps=data["timestamps"],
401
+ log_ids=data["log_ids"],
402
+ space_id=self._space_id,
403
+ )
404
+ self._has_local_buffer = True
405
+
406
+ def _persist_uploads_locally(self, uploads: list[UploadEntry]):
407
+ if not self._space_id:
408
+ return
409
+ for entry in uploads:
410
+ file_data = entry.get("uploaded_file")
411
+ file_path = ""
412
+ if isinstance(file_data, dict):
413
+ file_path = file_data.get("path", "")
414
+ elif hasattr(file_data, "path"):
415
+ file_path = str(file_data.path)
416
+ else:
417
+ file_path = str(file_data)
418
+ SQLiteStorage.add_pending_upload(
419
+ project=entry["project"],
420
+ space_id=self._space_id,
421
+ run_name=entry.get("run"),
422
+ step=entry.get("step"),
423
+ file_path=file_path,
424
+ relative_path=entry.get("relative_path"),
425
+ )
426
+ self._has_local_buffer = True
427
+
428
+ def _flush_local_buffer(self):
429
+ try:
430
+ buffered_logs = SQLiteStorage.get_pending_logs(self.project)
431
+ if buffered_logs:
432
+ self._client.predict(
433
+ api_name="/bulk_log",
434
+ logs=buffered_logs["logs"],
435
+ hf_token=huggingface_hub.utils.get_token(),
436
+ )
437
+ SQLiteStorage.clear_pending_logs(self.project, buffered_logs["ids"])
438
+
439
+ buffered_sys = SQLiteStorage.get_pending_system_logs(self.project)
440
+ if buffered_sys:
441
+ self._client.predict(
442
+ api_name="/bulk_log_system",
443
+ logs=buffered_sys["logs"],
444
+ hf_token=huggingface_hub.utils.get_token(),
445
+ )
446
+ SQLiteStorage.clear_pending_system_logs(
447
+ self.project, buffered_sys["ids"]
448
+ )
449
+
450
+ buffered_uploads = SQLiteStorage.get_pending_uploads(self.project)
451
+ if buffered_uploads:
452
+ upload_entries = []
453
+ for u in buffered_uploads["uploads"]:
454
+ fp = u["file_path"]
455
+ if Path(fp).exists():
456
+ upload_entries.append(
457
+ {
458
+ "project": u["project"],
459
+ "run": u["run"],
460
+ "step": u["step"],
461
+ "relative_path": u["relative_path"],
462
+ "uploaded_file": handle_file(fp),
463
+ }
464
+ )
465
+ if upload_entries:
466
+ self._client.predict(
467
+ api_name="/bulk_upload_media",
468
+ uploads=upload_entries,
469
+ hf_token=huggingface_hub.utils.get_token(),
470
+ )
471
+ SQLiteStorage.clear_pending_uploads(
472
+ self.project, buffered_uploads["ids"]
473
+ )
474
+
475
+ self._has_local_buffer = False
476
+ except Exception:
477
+ pass
478
+
479
+ def _init_client_background(self):
480
+ if self._client is None:
481
+ fib = utils.fibo()
482
+ for sleep_coefficient in fib:
483
+ if self._stop_flag.is_set():
484
+ break
485
+ try:
486
+ client = RemoteClient(
487
+ self.url,
488
+ hf_token=huggingface_hub.utils.get_token(),
489
+ verbose=False,
490
+ )
491
+
492
+ with self._client_lock:
493
+ self._client = client
494
+ break
495
+ except Exception:
496
+ pass
497
+ sleep_time = min(0.1 * sleep_coefficient, MAX_BACKOFF)
498
+ self._stop_flag.wait(timeout=sleep_time)
499
+
500
+ self._batch_sender()
501
+
502
+ def _queue_upload(
503
+ self,
504
+ file_path,
505
+ step: int | None,
506
+ relative_path: str | None = None,
507
+ use_run_name: bool = True,
508
+ ):
509
+ if self._is_local:
510
+ self._save_upload_locally(file_path, step, relative_path, use_run_name)
511
+ else:
512
+ upload_entry: UploadEntry = {
513
+ "project": self.project,
514
+ "run": self.name if use_run_name else None,
515
+ "step": step,
516
+ "relative_path": relative_path,
517
+ "uploaded_file": handle_file(file_path),
518
+ }
519
+ with self._client_lock:
520
+ self._queued_uploads.append(upload_entry)
521
+
522
+ def _save_upload_locally(
523
+ self,
524
+ file_path,
525
+ step: int | None,
526
+ relative_path: str | None = None,
527
+ use_run_name: bool = True,
528
+ ):
529
+ media_path = get_project_media_path(
530
+ project=self.project,
531
+ run=self.name if use_run_name else None,
532
+ step=step,
533
+ relative_path=relative_path,
534
+ )
535
+ src = Path(file_path)
536
+ if src.exists() and str(src.resolve()) != str(Path(media_path).resolve()):
537
+ shutil.copy(str(src), str(media_path))
538
+
539
+ def _process_media(self, value: TrackioMedia, step: int | None) -> dict:
540
+ value._save(self.project, self.name, step if step is not None else 0)
541
+ if self._space_id:
542
+ self._queue_upload(value._get_absolute_file_path(), step)
543
+ return value._to_dict()
544
+
545
+ def _scan_and_queue_media_uploads(self, table_dict: dict, step: int | None):
546
+ if not self._space_id:
547
+ return
548
+
549
+ table_data = table_dict.get("_value", [])
550
+ for row in table_data:
551
+ for value in row.values():
552
+ if isinstance(value, dict) and value.get("_type") in [
553
+ "trackio.image",
554
+ "trackio.video",
555
+ "trackio.audio",
556
+ ]:
557
+ file_path = value.get("file_path")
558
+ if file_path:
559
+ from trackio.utils import MEDIA_DIR
560
+
561
+ absolute_path = MEDIA_DIR / file_path
562
+ self._queue_upload(absolute_path, step)
563
+ elif isinstance(value, list):
564
+ for item in value:
565
+ if isinstance(item, dict) and item.get("_type") in [
566
+ "trackio.image",
567
+ "trackio.video",
568
+ "trackio.audio",
569
+ ]:
570
+ file_path = item.get("file_path")
571
+ if file_path:
572
+ from trackio.utils import MEDIA_DIR
573
+
574
+ absolute_path = MEDIA_DIR / file_path
575
+ self._queue_upload(absolute_path, step)
576
+
577
+ def _ensure_sender_alive(self):
578
+ if self._is_local:
579
+ if (
580
+ hasattr(self, "_local_sender_thread")
581
+ and not self._local_sender_thread.is_alive()
582
+ and not self._stop_flag.is_set()
583
+ ):
584
+ self._local_sender_thread = threading.Thread(
585
+ target=self._local_batch_sender
586
+ )
587
+ self._local_sender_thread.daemon = True
588
+ self._local_sender_thread.start()
589
+ else:
590
+ if (
591
+ self._client_thread is not None
592
+ and not self._client_thread.is_alive()
593
+ and not self._stop_flag.is_set()
594
+ ):
595
+ self._client_thread = threading.Thread(
596
+ target=self._init_client_background
597
+ )
598
+ self._client_thread.daemon = True
599
+ self._client_thread.start()
600
+
601
+ def log(self, metrics: dict, step: int | None = None):
602
+ renamed_keys = []
603
+ new_metrics = {}
604
+
605
+ for k, v in metrics.items():
606
+ if k in utils.RESERVED_KEYS or k.startswith("__"):
607
+ new_key = f"__{k}"
608
+ renamed_keys.append(k)
609
+ new_metrics[new_key] = v
610
+ else:
611
+ new_metrics[k] = v
612
+
613
+ if renamed_keys:
614
+ warnings.warn(f"Reserved keys renamed: {renamed_keys} → '__{{key}}'")
615
+
616
+ metrics = new_metrics
617
+ for key, value in metrics.items():
618
+ if isinstance(value, Table):
619
+ metrics[key] = value._to_dict(
620
+ project=self.project, run=self.name, step=step
621
+ )
622
+ self._scan_and_queue_media_uploads(metrics[key], step)
623
+ elif isinstance(value, Histogram):
624
+ metrics[key] = value._to_dict()
625
+ elif isinstance(value, Markdown):
626
+ metrics[key] = value._to_dict()
627
+ elif isinstance(value, TrackioMedia):
628
+ metrics[key] = self._process_media(value, step)
629
+ metrics = utils.serialize_values(metrics)
630
+
631
+ if step is None:
632
+ step = self._next_step
633
+ self._next_step = max(self._next_step, step + 1)
634
+
635
+ config_to_log = None
636
+ if not self._config_logged and self.config:
637
+ config_to_log = utils.to_json_safe(self.config)
638
+ self._config_logged = True
639
+
640
+ log_entry: LogEntry = {
641
+ "project": self.project,
642
+ "run": self.name,
643
+ "metrics": metrics,
644
+ "step": step,
645
+ "config": config_to_log,
646
+ "log_id": uuid.uuid4().hex,
647
+ }
648
+
649
+ with self._client_lock:
650
+ self._queued_logs.append(log_entry)
651
+ self._ensure_sender_alive()
652
+
653
+ def alert(
654
+ self,
655
+ title: str,
656
+ text: str | None = None,
657
+ level: AlertLevel = AlertLevel.WARN,
658
+ step: int | None = None,
659
+ webhook_url: str | None = None,
660
+ ):
661
+ if step is None:
662
+ step = max(self._next_step - 1, 0)
663
+ timestamp = datetime.now(timezone.utc).isoformat()
664
+
665
+ print(format_alert_terminal(level, title, text, step))
666
+
667
+ alert_entry: AlertEntry = {
668
+ "project": self.project,
669
+ "run": self.name,
670
+ "title": title,
671
+ "text": text,
672
+ "level": level.value,
673
+ "step": step,
674
+ "timestamp": timestamp,
675
+ "alert_id": uuid.uuid4().hex,
676
+ }
677
+
678
+ with self._client_lock:
679
+ self._queued_alerts.append(alert_entry)
680
+ self._ensure_sender_alive()
681
+
682
+ url = webhook_url or self._webhook_url
683
+ if url and should_send_webhook(level, self._webhook_min_level):
684
+ t = threading.Thread(
685
+ target=send_webhook,
686
+ args=(
687
+ url,
688
+ level,
689
+ title,
690
+ text,
691
+ self.project,
692
+ self.name,
693
+ step,
694
+ timestamp,
695
+ ),
696
+ daemon=True,
697
+ )
698
+ t.start()
699
+
700
+ def log_system(self, metrics: dict):
701
+ metrics = utils.serialize_values(metrics)
702
+ timestamp = datetime.now(timezone.utc).isoformat()
703
+
704
+ system_log_entry: SystemLogEntry = {
705
+ "project": self.project,
706
+ "run": self.name,
707
+ "metrics": metrics,
708
+ "timestamp": timestamp,
709
+ "log_id": uuid.uuid4().hex,
710
+ }
711
+
712
+ with self._client_lock:
713
+ self._queued_system_logs.append(system_log_entry)
714
+ self._ensure_sender_alive()
715
+
716
+ def finish(self):
717
+ if self._gpu_monitor is not None:
718
+ self._gpu_monitor.stop()
719
+
720
+ self._stop_flag.set()
721
+
722
+ if self._is_local:
723
+ if hasattr(self, "_local_sender_thread"):
724
+ print("* Run finished. Uploading logs to Trackio (please wait...)")
725
+ self._local_sender_thread.join(timeout=30)
726
+ if self._local_sender_thread.is_alive():
727
+ warnings.warn(
728
+ "Could not flush all logs within 30s. Some data may be buffered locally."
729
+ )
730
+ else:
731
+ if self._client_thread is not None:
732
+ print(
733
+ "* Run finished. Uploading logs to Trackio Space (please wait...)"
734
+ )
735
+ self._client_thread.join(timeout=30)
736
+ if self._client_thread.is_alive():
737
+ warnings.warn(
738
+ "Could not flush all logs within 30s. Some data may be buffered locally."
739
+ )
740
+ if SQLiteStorage.has_pending_data(self.project):
741
+ warnings.warn(
742
+ f"* Some logs could not be sent to the Space (it may still be starting up). "
743
+ f"They have been saved locally and will be sent automatically next time you call: "
744
+ f'trackio.init(project="{self.project}", space_id="{self._space_id}")'
745
+ )