Spaces:
Configuration error
Configuration error
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +2 -0
- trackio/.DS_Store +0 -0
- trackio/CHANGELOG.md +232 -0
- trackio/__init__.py +750 -0
- trackio/_vendor/__init__.py +0 -0
- trackio/_vendor/gradio_exceptions.py +6 -0
- trackio/_vendor/networking.py +81 -0
- trackio/_vendor/tunneling.py +191 -0
- trackio/alerts.py +184 -0
- trackio/api.py +87 -0
- trackio/apple_gpu.py +253 -0
- trackio/asgi_app.py +175 -0
- trackio/assets/badge.png +3 -0
- trackio/assets/trackio_logo_dark.png +0 -0
- trackio/assets/trackio_logo_light.png +0 -0
- trackio/assets/trackio_logo_old.png +3 -0
- trackio/assets/trackio_logo_type_dark.png +0 -0
- trackio/assets/trackio_logo_type_dark_transparent.png +0 -0
- trackio/assets/trackio_logo_type_light.png +0 -0
- trackio/assets/trackio_logo_type_light_transparent.png +0 -0
- trackio/bucket_storage.py +151 -0
- trackio/cli.py +1272 -0
- trackio/cli_helpers.py +158 -0
- trackio/commit_scheduler.py +310 -0
- trackio/context_vars.py +18 -0
- trackio/deploy.py +938 -0
- trackio/dummy_commit_scheduler.py +19 -0
- trackio/exceptions.py +2 -0
- trackio/frontend/dist/assets/index-Cs1UkfOV.css +1 -0
- trackio/frontend/dist/assets/index-UYQ1xITV.js +0 -0
- trackio/frontend/dist/index.html +14 -0
- trackio/frontend/eslint.config.js +42 -0
- trackio/frontend/index.html +13 -0
- trackio/frontend_server.py +63 -0
- trackio/gpu.py +357 -0
- trackio/histogram.py +71 -0
- trackio/imports.py +290 -0
- trackio/launch.py +211 -0
- trackio/launch_utils.py +33 -0
- trackio/markdown.py +21 -0
- trackio/mcp_setup.py +6 -0
- trackio/media/__init__.py +27 -0
- trackio/media/audio.py +167 -0
- trackio/media/image.py +84 -0
- trackio/media/media.py +79 -0
- trackio/media/utils.py +60 -0
- trackio/media/video.py +246 -0
- trackio/package.json +6 -0
- trackio/py.typed +0 -0
- trackio/remote_client.py +28 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
trackio/assets/badge.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
trackio/assets/trackio_logo_old.png filter=lfs diff=lfs merge=lfs -text
|
trackio/.DS_Store
ADDED
|
Binary file (8.2 kB). View file
|
|
|
trackio/CHANGELOG.md
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# trackio
|
| 2 |
+
|
| 3 |
+
## 0.22.0
|
| 4 |
+
|
| 5 |
+
### Features
|
| 6 |
+
|
| 7 |
+
- [#484](https://github.com/gradio-app/trackio/pull/484) [`cc05ada`](https://github.com/gradio-app/trackio/commit/cc05ada8e89773f3a894af99b801ef680f64418f) - Fix duplicate columns in parquet export. Thanks @abidlabs!
|
| 8 |
+
- [#487](https://github.com/gradio-app/trackio/pull/487) [`853f764`](https://github.com/gradio-app/trackio/commit/853f7646a70d12633afaa4f69db86425aa665413) - Relax `PIL` dependency and remove `plotly` as it's no longer used. Thanks @abidlabs!
|
| 9 |
+
|
| 10 |
+
## 0.21.2
|
| 11 |
+
|
| 12 |
+
### Features
|
| 13 |
+
|
| 14 |
+
- [#482](https://github.com/gradio-app/trackio/pull/482) [`f62180a`](https://github.com/gradio-app/trackio/commit/f62180a0218bc99a259d5ca110a0384a6cae11c8) - Use server-side bucket copy when freezing Spaces. Thanks @abidlabs!
|
| 15 |
+
|
| 16 |
+
## 0.21.1
|
| 17 |
+
|
| 18 |
+
### Features
|
| 19 |
+
|
| 20 |
+
- [#475](https://github.com/gradio-app/trackio/pull/475) [`fcb476c`](https://github.com/gradio-app/trackio/commit/fcb476cd37a40923e9679aaf966f41d582a878a8) - Tweaks. Thanks @abidlabs!
|
| 21 |
+
- [#477](https://github.com/gradio-app/trackio/pull/477) [`7d52dfd`](https://github.com/gradio-app/trackio/commit/7d52dfdce5b6eff6a34501a6d5a620220663cf09) - Fix `.sync()` and add `.freeze()` as a separate methods. Thanks @abidlabs!
|
| 22 |
+
|
| 23 |
+
## 0.21.0
|
| 24 |
+
|
| 25 |
+
### Features
|
| 26 |
+
|
| 27 |
+
- [#467](https://github.com/gradio-app/trackio/pull/467) [`f357deb`](https://github.com/gradio-app/trackio/commit/f357debf78957e4c1f2b901bee4f77cf397298b4) - Allow logged metrics as x-axis choices. Thanks @abidlabs!
|
| 28 |
+
- [#474](https://github.com/gradio-app/trackio/pull/474) [`655673d`](https://github.com/gradio-app/trackio/commit/655673d4c6b7c8b7ee8f87f2589f2dbbc3d2ef91) - Fix file descriptor leak from `sqlite3.connect`. Thanks @abidlabs!
|
| 29 |
+
- [#470](https://github.com/gradio-app/trackio/pull/470) [`bea8c9d`](https://github.com/gradio-app/trackio/commit/bea8c9dcae0b59d071b6c779c97ee525c9bbf6e7) - Restores tooltips to line plots and fixes the call to uses TTL instead of OAuth. Thanks @abidlabs!
|
| 30 |
+
- [#471](https://github.com/gradio-app/trackio/pull/471) [`246fce0`](https://github.com/gradio-app/trackio/commit/246fce0a01619e1c2c538c67b3e460883334d500) - Deprecate dataset backend in favor of buckets. Thanks @abidlabs!
|
| 31 |
+
- [#465](https://github.com/gradio-app/trackio/pull/465) [`3e11174`](https://github.com/gradio-app/trackio/commit/3e1117438bb8168b802245a33059affa558ae519) - Use HF buckets as backend. Thanks @abidlabs!
|
| 32 |
+
- [#469](https://github.com/gradio-app/trackio/pull/469) [`915d170`](https://github.com/gradio-app/trackio/commit/915d17045133172b59195acfdcc70709229668aa) - Make static Spaces work with Buckets and also allow conversion from Gradio SDK to Static Spaces. Thanks @abidlabs!
|
| 33 |
+
|
| 34 |
+
## 0.20.2
|
| 35 |
+
|
| 36 |
+
### Features
|
| 37 |
+
|
| 38 |
+
- [#464](https://github.com/gradio-app/trackio/pull/464) [`c89ebb3`](https://github.com/gradio-app/trackio/commit/c89ebb3b50f695bc7f16cbc6f46dce86f79a01e9) - Improve rendering of curves. Thanks @abidlabs!
|
| 39 |
+
- [#462](https://github.com/gradio-app/trackio/pull/462) [`9160b78`](https://github.com/gradio-app/trackio/commit/9160b78ff6f258f0b87a4f34a24e7d0b5dfbf2fb) - Refactor plot title to display only the metric name without the path. Thanks @qgallouedec!
|
| 40 |
+
|
| 41 |
+
## 0.20.1
|
| 42 |
+
|
| 43 |
+
### Features
|
| 44 |
+
|
| 45 |
+
- [#454](https://github.com/gradio-app/trackio/pull/454) [`22881db`](https://github.com/gradio-app/trackio/commit/22881dbbbb6b81197a00a19853771007093d61e4) - Bar chart single point. Thanks @abidlabs!
|
| 46 |
+
- [#455](https://github.com/gradio-app/trackio/pull/455) [`f8db51a`](https://github.com/gradio-app/trackio/commit/f8db51a20ca61ef703f3f2c2ee1ebd9c4f239cf2) - Adds a static Trackio mode via `trackio.sync(sdk="static")` and support for the `TRACKIO_SPACE_ID` environment variable. Thanks @abidlabs!
|
| 47 |
+
|
| 48 |
+
## 0.20.0
|
| 49 |
+
|
| 50 |
+
### Features
|
| 51 |
+
|
| 52 |
+
- [#450](https://github.com/gradio-app/trackio/pull/450) [`b0571ef`](https://github.com/gradio-app/trackio/commit/b0571ef6207a1ce346696f858ad2b7b584dd194f) - Use Svelte source for Gradio components directly in Trackio dashboard. Thanks @abidlabs!
|
| 53 |
+
|
| 54 |
+
## 0.19.0
|
| 55 |
+
|
| 56 |
+
### Features
|
| 57 |
+
|
| 58 |
+
- [#445](https://github.com/gradio-app/trackio/pull/445) [`cef4a58`](https://github.com/gradio-app/trackio/commit/cef4a583cb76f4091fc6c0e5783124ee84f8e243) - Add remote HF Space support to CLI. Thanks @abidlabs!
|
| 59 |
+
- [#444](https://github.com/gradio-app/trackio/pull/444) [`358f2a9`](https://github.com/gradio-app/trackio/commit/358f2a9ca238ee8b90b5a8c96220da287e0698fb) - Fix alerts placeholder flashing on reports page. Thanks @abidlabs!
|
| 60 |
+
|
| 61 |
+
## 0.18.0
|
| 62 |
+
|
| 63 |
+
### Features
|
| 64 |
+
|
| 65 |
+
- [#435](https://github.com/gradio-app/trackio/pull/435) [`4a47112`](https://github.com/gradio-app/trackio/commit/4a471128e18a39e45fad48a67fd711c5ae9e4aed) - feat: allow hiding section header accordions. Thanks @Saba9!
|
| 66 |
+
- [#439](https://github.com/gradio-app/trackio/pull/439) [`18e9650`](https://github.com/gradio-app/trackio/commit/18e96503d5a3a7cf926e92782d457e23c19942bd) - Add alerts with webhooks, CLI, and documentation. Thanks @abidlabs!
|
| 67 |
+
- [#438](https://github.com/gradio-app/trackio/pull/438) [`0875ccd`](https://github.com/gradio-app/trackio/commit/0875ccd3d8a41b1376f64030f21cfe8cdcc73b05) - Add "share this view" functionality. Thanks @qgallouedec!
|
| 68 |
+
- [#409](https://github.com/gradio-app/trackio/pull/409) [`9282403`](https://github.com/gradio-app/trackio/commit/9282403d8896d48679b0f888208a7ba5bdd4271a) - Add Apple Silicon GPU and system monitoring support. Thanks @znation!
|
| 69 |
+
- [#434](https://github.com/gradio-app/trackio/pull/434) [`4193223`](https://github.com/gradio-app/trackio/commit/41932230a3a2e1c16405dba08ecba5a42f11d1a8) - fix: table slider crash. Thanks @Saba9!
|
| 70 |
+
|
| 71 |
+
### Fixes
|
| 72 |
+
|
| 73 |
+
- [#441](https://github.com/gradio-app/trackio/pull/441) [`3a2d11d`](https://github.com/gradio-app/trackio/commit/3a2d11dab0b4b37c925abc30ef84b0e2910321ee) - preserve x-axis step when toggling run checkboxes. Thanks @Saba9!
|
| 74 |
+
|
| 75 |
+
## 0.17.0
|
| 76 |
+
|
| 77 |
+
### Features
|
| 78 |
+
|
| 79 |
+
- [#428](https://github.com/gradio-app/trackio/pull/428) [`f7dd1ce`](https://github.com/gradio-app/trackio/commit/f7dd1ce2dc8a1936f9983467fcbcf93bfef01e09) - feat: add ability to rename runs. Thanks @Saba9!
|
| 80 |
+
- [#437](https://github.com/gradio-app/trackio/pull/437) [`2727c0b`](https://github.com/gradio-app/trackio/commit/2727c0b0755f48f7f186162ea45185c98f6b5516) - Add markdown reports across Trackio. Thanks @abidlabs!
|
| 81 |
+
- [#427](https://github.com/gradio-app/trackio/pull/427) [`5aeb9ed`](https://github.com/gradio-app/trackio/commit/5aeb9edcfd2068d309d9d64f172dcbcc327be1ab) - Make Trackio logging much more robust. Thanks @abidlabs!
|
| 82 |
+
|
| 83 |
+
## 0.16.1
|
| 84 |
+
|
| 85 |
+
### Features
|
| 86 |
+
|
| 87 |
+
- [#431](https://github.com/gradio-app/trackio/pull/431) [`c7ce55b`](https://github.com/gradio-app/trackio/commit/c7ce55b14dd5eb0c2165fb15df17dd60721c9325) - Lazy load the UI when trackio is imported. Thanks @abidlabs!
|
| 88 |
+
|
| 89 |
+
## 0.16.0
|
| 90 |
+
|
| 91 |
+
### Features
|
| 92 |
+
|
| 93 |
+
- [#426](https://github.com/gradio-app/trackio/pull/426) [`ead4dc8`](https://github.com/gradio-app/trackio/commit/ead4dc8e74ee2d8e47d61bca0a7668456acf49be) - Fix redundant double rendering of group checkboxes. Thanks @abidlabs!
|
| 94 |
+
- [#413](https://github.com/gradio-app/trackio/pull/413) [`39c4750`](https://github.com/gradio-app/trackio/commit/39c4750951d554ba6eb4d58847c6bb444b2891a8) - Check `dist-packages` when checking for source installation. Thanks @sergiopaniego!
|
| 95 |
+
- [#423](https://github.com/gradio-app/trackio/pull/423) [`2e52ab3`](https://github.com/gradio-app/trackio/commit/2e52ab303e3041718a6a56fbf84d0848aca9ad67) - Fix legend outline visibility issue. Thanks @Raghunath-Balaji!
|
| 96 |
+
- [#407](https://github.com/gradio-app/trackio/pull/407) [`c8a384d`](https://github.com/gradio-app/trackio/commit/c8a384ddfe5a295cecf862a26178d40e48acb424) - Fix pytests that were failling locally on MacOS. Thanks @abidlabs!
|
| 97 |
+
- [#405](https://github.com/gradio-app/trackio/pull/405) [`35aae4e`](https://github.com/gradio-app/trackio/commit/35aae4e3aa3e2b2888887528478b9dc6a9808bda) - Add conditional padding for HF Space dashboard when not in iframe. Thanks @znation!
|
| 98 |
+
|
| 99 |
+
## 0.15.0
|
| 100 |
+
|
| 101 |
+
### Features
|
| 102 |
+
|
| 103 |
+
- [#397](https://github.com/gradio-app/trackio/pull/397) [`6b38ad0`](https://github.com/gradio-app/trackio/commit/6b38ad02e5d73a0df49c4eede7e91331282ece04) - Adds `--host` cli option support. Thanks @abidlabs!
|
| 104 |
+
- [#396](https://github.com/gradio-app/trackio/pull/396) [`4a4d1ab`](https://github.com/gradio-app/trackio/commit/4a4d1ab85e63d923132a3fa7afa5d90e16431bec) - Fix run selection issue. Thanks @abidlabs!
|
| 105 |
+
- [#394](https://github.com/gradio-app/trackio/pull/394) [`c47a3a3`](https://github.com/gradio-app/trackio/commit/c47a3a31f8c4b83bce1aa7fc22eeba3d9021ad3d) - Add wandb-compatible API for trackio. Thanks @abidlabs!
|
| 106 |
+
- [#378](https://github.com/gradio-app/trackio/pull/378) [`b02046a`](https://github.com/gradio-app/trackio/commit/b02046a5b0dad7c9854e099a87f884afba4aecb2) - Add JSON export button for line plots and upgrade gradio dependency. Thanks @JamshedAli18!
|
| 107 |
+
|
| 108 |
+
## 0.14.2
|
| 109 |
+
|
| 110 |
+
### Features
|
| 111 |
+
|
| 112 |
+
- [#386](https://github.com/gradio-app/trackio/pull/386) [`f9452cd`](https://github.com/gradio-app/trackio/commit/f9452cdb8f0819368f3610f7ac0ed08957305275) - Fixing some issues related to deployed Trackio Spaces. Thanks @abidlabs!
|
| 113 |
+
|
| 114 |
+
## 0.14.1
|
| 115 |
+
|
| 116 |
+
### Features
|
| 117 |
+
|
| 118 |
+
- [#382](https://github.com/gradio-app/trackio/pull/382) [`44fe9bb`](https://github.com/gradio-app/trackio/commit/44fe9bb264fb2aafb0ec302ff15227c045819a2c) - Fix app file path when Trackio is not installed from source. Thanks @abidlabs!
|
| 119 |
+
- [#380](https://github.com/gradio-app/trackio/pull/380) [`c3f4cff`](https://github.com/gradio-app/trackio/commit/c3f4cff74bc5676e812773d8571454894fcdc7cc) - Add CLI commands for querying projects, runs, and metrics. Thanks @abidlabs!
|
| 120 |
+
|
| 121 |
+
## 0.14.0
|
| 122 |
+
|
| 123 |
+
### Features
|
| 124 |
+
|
| 125 |
+
- [#377](https://github.com/gradio-app/trackio/pull/377) [`5c5015b`](https://github.com/gradio-app/trackio/commit/5c5015b68c85c5de51111dad983f735c27b9a05f) - fixed wrapping issue in Runs table. Thanks @gaganchapa!
|
| 126 |
+
- [#374](https://github.com/gradio-app/trackio/pull/374) [`388e26b`](https://github.com/gradio-app/trackio/commit/388e26b9e9f24cd7ad203affe9b709be885b3d24) - Save Optimized Parquet files. Thanks @lhoestq!
|
| 127 |
+
- [#371](https://github.com/gradio-app/trackio/pull/371) [`fbace9c`](https://github.com/gradio-app/trackio/commit/fbace9cd7732c166f34d268f54b05bb06846cc5d) - Add GPU metrics logging. Thanks @kashif!
|
| 128 |
+
- [#367](https://github.com/gradio-app/trackio/pull/367) [`862840c`](https://github.com/gradio-app/trackio/commit/862840c13e30fc960cbee5b9eac4d3c25beba9de) - Add option to only show latest run, and fix the double logo issue. Thanks @abidlabs!
|
| 129 |
+
|
| 130 |
+
## 0.13.1
|
| 131 |
+
|
| 132 |
+
### Features
|
| 133 |
+
|
| 134 |
+
- [#369](https://github.com/gradio-app/trackio/pull/369) [`767e9fe`](https://github.com/gradio-app/trackio/commit/767e9fe095d7c6ed102016caf927c1517fb8618c) - tiny pr removing unnecessary code. Thanks @abidlabs!
|
| 135 |
+
|
| 136 |
+
## 0.13.0
|
| 137 |
+
|
| 138 |
+
### Features
|
| 139 |
+
|
| 140 |
+
- [#358](https://github.com/gradio-app/trackio/pull/358) [`073715d`](https://github.com/gradio-app/trackio/commit/073715d1caf8282f68890117f09c3ac301205312) - Improvements to `trackio.sync()`. Thanks @abidlabs!
|
| 141 |
+
|
| 142 |
+
## 0.12.0
|
| 143 |
+
|
| 144 |
+
### Features
|
| 145 |
+
|
| 146 |
+
- [#357](https://github.com/gradio-app/trackio/pull/357) [`02ba815`](https://github.com/gradio-app/trackio/commit/02ba815358060f1966052de051a5bdb09702920e) - Redesign media and tables to show up on separate page. Thanks @abidlabs!
|
| 147 |
+
- [#359](https://github.com/gradio-app/trackio/pull/359) [`08fe9c9`](https://github.com/gradio-app/trackio/commit/08fe9c9ddd7fe99ee811555fdfb62df9ab88e939) - docs: Improve docstrings. Thanks @qgallouedec!
|
| 148 |
+
|
| 149 |
+
## 0.11.0
|
| 150 |
+
|
| 151 |
+
### Features
|
| 152 |
+
|
| 153 |
+
- [#355](https://github.com/gradio-app/trackio/pull/355) [`ea51f49`](https://github.com/gradio-app/trackio/commit/ea51f4954922f21be76ef828700420fe9a912c4b) - Color code run checkboxes and match with plot lines. Thanks @abidlabs!
|
| 154 |
+
- [#353](https://github.com/gradio-app/trackio/pull/353) [`8abe691`](https://github.com/gradio-app/trackio/commit/8abe6919aeefe21fc7a23af814883efbb037c21f) - Remove show_api from demo.launch. Thanks @sergiopaniego!
|
| 155 |
+
- [#351](https://github.com/gradio-app/trackio/pull/351) [`8a8957e`](https://github.com/gradio-app/trackio/commit/8a8957e530dd7908d1fef7f2df030303f808101f) - Add `trackio.save()`. Thanks @abidlabs!
|
| 156 |
+
|
| 157 |
+
## 0.10.0
|
| 158 |
+
|
| 159 |
+
### Features
|
| 160 |
+
|
| 161 |
+
- [#305](https://github.com/gradio-app/trackio/pull/305) [`e64883a`](https://github.com/gradio-app/trackio/commit/e64883a51f7b8b93f7d48b8afe55acdb62238b71) - bump to gradio 6.0, make `trackio` compatible, and fix related issues. Thanks @abidlabs!
|
| 162 |
+
|
| 163 |
+
## 0.9.1
|
| 164 |
+
|
| 165 |
+
### Features
|
| 166 |
+
|
| 167 |
+
- [#344](https://github.com/gradio-app/trackio/pull/344) [`7e01024`](https://github.com/gradio-app/trackio/commit/7e010241d9a34794e0ce0dc19c1a6f0cf94ba856) - Avoid redundant calls to /whoami-v2. Thanks @Wauplin!
|
| 168 |
+
|
| 169 |
+
## 0.9.0
|
| 170 |
+
|
| 171 |
+
### Features
|
| 172 |
+
|
| 173 |
+
- [#343](https://github.com/gradio-app/trackio/pull/343) [`51bea30`](https://github.com/gradio-app/trackio/commit/51bea30f2877adff8e6497466d3a799400a0a049) - Sync offline projects to Hugging Face spaces. Thanks @candemircan!
|
| 174 |
+
- [#341](https://github.com/gradio-app/trackio/pull/341) [`4fd841f`](https://github.com/gradio-app/trackio/commit/4fd841fa190e15071b02f6fba7683ef4f393a654) - Adds a basic UI test to `trackio`. Thanks @abidlabs!
|
| 175 |
+
- [#339](https://github.com/gradio-app/trackio/pull/339) [`011d91b`](https://github.com/gradio-app/trackio/commit/011d91bb6ae266516fd250a349285670a8049d05) - Allow customzing the trackio color palette. Thanks @abidlabs!
|
| 176 |
+
|
| 177 |
+
## 0.8.1
|
| 178 |
+
|
| 179 |
+
### Features
|
| 180 |
+
|
| 181 |
+
- [#336](https://github.com/gradio-app/trackio/pull/336) [`5f9f51d`](https://github.com/gradio-app/trackio/commit/5f9f51dac8677f240d7c42c3e3b2660a22aee138) - Support a list of `Trackio.Image` in a `trackio.Table` cell. Thanks @abidlabs!
|
| 182 |
+
|
| 183 |
+
## 0.8.0
|
| 184 |
+
|
| 185 |
+
### Features
|
| 186 |
+
|
| 187 |
+
- [#331](https://github.com/gradio-app/trackio/pull/331) [`2c02d0f`](https://github.com/gradio-app/trackio/commit/2c02d0fd0a5824160528782402bb0dd4083396d5) - Truncate table string values that are greater than 250 characters (configuirable via env variable). Thanks @abidlabs!
|
| 188 |
+
- [#324](https://github.com/gradio-app/trackio/pull/324) [`50b2122`](https://github.com/gradio-app/trackio/commit/50b2122e7965ac82a72e6cb3b7d048bc10a2a6b1) - Add log y-axis functionality to UI. Thanks @abidlabs!
|
| 189 |
+
- [#326](https://github.com/gradio-app/trackio/pull/326) [`61dc1f4`](https://github.com/gradio-app/trackio/commit/61dc1f40af2f545f8e70395ddf0dbb8aee6b60d5) - Fix: improve table rendering for metrics in Trackio Dashboard. Thanks @vigneshwaran!
|
| 190 |
+
- [#328](https://github.com/gradio-app/trackio/pull/328) [`6857cbb`](https://github.com/gradio-app/trackio/commit/6857cbbe557a59a4642f210ec42566d108294e63) - Support trackio.Table with trackio.Image columns. Thanks @abidlabs!
|
| 191 |
+
- [#323](https://github.com/gradio-app/trackio/pull/323) [`6857cbb`](https://github.com/gradio-app/trackio/commit/6857cbbe557a59a4642f210ec42566d108294e63) - add Trackio client implementations in Go, Rust, and JS. Thanks @vaibhav-research!
|
| 192 |
+
|
| 193 |
+
## 0.7.0
|
| 194 |
+
|
| 195 |
+
### Features
|
| 196 |
+
|
| 197 |
+
- [#277](https://github.com/gradio-app/trackio/pull/277) [`db35601`](https://github.com/gradio-app/trackio/commit/db35601b9c023423c4654c9909b8ab73e58737de) - fix: make grouped runs view reflect live updates. Thanks @Saba9!
|
| 198 |
+
- [#320](https://github.com/gradio-app/trackio/pull/320) [`24ae739`](https://github.com/gradio-app/trackio/commit/24ae73969b09fb3126acd2f91647cdfbf8cf72a1) - Add additional query parms for xmin, xmax, and smoothing. Thanks @abidlabs!
|
| 199 |
+
- [#270](https://github.com/gradio-app/trackio/pull/270) [`cd1dfc3`](https://github.com/gradio-app/trackio/commit/cd1dfc3dc641b4499ac6d4a1b066fa8e2b52c57b) - feature: add support for logging audio. Thanks @Saba9!
|
| 200 |
+
|
| 201 |
+
## 0.6.0
|
| 202 |
+
|
| 203 |
+
### Features
|
| 204 |
+
|
| 205 |
+
- [#309](https://github.com/gradio-app/trackio/pull/309) [`1df2353`](https://github.com/gradio-app/trackio/commit/1df23534d6c01938c8db9c0f584ffa23e8d6021d) - Add histogram support with wandb-compatible API. Thanks @abidlabs!
|
| 206 |
+
- [#315](https://github.com/gradio-app/trackio/pull/315) [`76ba060`](https://github.com/gradio-app/trackio/commit/76ba06055dc43ca8f03b79f3e72d761949bd19a8) - Add guards to avoid silent fails. Thanks @Xmaster6y!
|
| 207 |
+
- [#313](https://github.com/gradio-app/trackio/pull/313) [`a606b3e`](https://github.com/gradio-app/trackio/commit/a606b3e1c5edf3d4cf9f31bd50605226a5a1c5d0) - No longer prevent certain keys from being used. Instead, dunderify them to prevent collisions with internal usage. Thanks @abidlabs!
|
| 208 |
+
- [#317](https://github.com/gradio-app/trackio/pull/317) [`27370a5`](https://github.com/gradio-app/trackio/commit/27370a595d0dbdf7eebbe7159d2ba778f039da44) - quick fixes for trackio.histogram. Thanks @abidlabs!
|
| 209 |
+
- [#312](https://github.com/gradio-app/trackio/pull/312) [`aa0f3bf`](https://github.com/gradio-app/trackio/commit/aa0f3bf372e7a0dd592a38af699c998363830eeb) - Fix video logging by adding TRACKIO_DIR to allowed_paths. Thanks @abidlabs!
|
| 210 |
+
|
| 211 |
+
## 0.5.3
|
| 212 |
+
|
| 213 |
+
### Features
|
| 214 |
+
|
| 215 |
+
- [#300](https://github.com/gradio-app/trackio/pull/300) [`5e4cacf`](https://github.com/gradio-app/trackio/commit/5e4cacf2e7ce527b4ce60de3a5bc05d2c02c77fb) - Adds more environment variables to allow customization of Trackio dashboard. Thanks @abidlabs!
|
| 216 |
+
|
| 217 |
+
## 0.5.2
|
| 218 |
+
|
| 219 |
+
### Features
|
| 220 |
+
|
| 221 |
+
- [#293](https://github.com/gradio-app/trackio/pull/293) [`64afc28`](https://github.com/gradio-app/trackio/commit/64afc28d3ea1dfd821472dc6bf0b8ed35a9b74be) - Ensures that the TRACKIO_DIR environment variable is respected. Thanks @abidlabs!
|
| 222 |
+
- [#287](https://github.com/gradio-app/trackio/pull/287) [`cd3e929`](https://github.com/gradio-app/trackio/commit/cd3e9294320949e6b8b829239069a43d5d7ff4c1) - fix(sqlite): unify .sqlite extension, allow export when DBs exist, clean WAL sidecars on import. Thanks @vaibhav-research!
|
| 223 |
+
|
| 224 |
+
### Fixes
|
| 225 |
+
|
| 226 |
+
- [#291](https://github.com/gradio-app/trackio/pull/291) [`3b5adc3`](https://github.com/gradio-app/trackio/commit/3b5adc3d1f452dbab7a714d235f4974782f93730) - Fix the wheel build. Thanks @pngwn!
|
| 227 |
+
|
| 228 |
+
## 0.5.1
|
| 229 |
+
|
| 230 |
+
### Fixes
|
| 231 |
+
|
| 232 |
+
- [#278](https://github.com/gradio-app/trackio/pull/278) [`314c054`](https://github.com/gradio-app/trackio/commit/314c05438007ddfea3383e06fd19143e27468e2d) - Fix row orientation of metrics plots. Thanks @abidlabs!
|
trackio/__init__.py
ADDED
|
@@ -0,0 +1,750 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import atexit
|
| 2 |
+
import glob
|
| 3 |
+
import json
|
| 4 |
+
import logging
|
| 5 |
+
import os
|
| 6 |
+
import shutil
|
| 7 |
+
import warnings
|
| 8 |
+
import webbrowser
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
import huggingface_hub
|
| 13 |
+
from gradio_client import Client, handle_file
|
| 14 |
+
from huggingface_hub import SpaceStorage
|
| 15 |
+
from huggingface_hub.errors import LocalTokenNotFoundError
|
| 16 |
+
|
| 17 |
+
from trackio import context_vars, deploy, utils
|
| 18 |
+
from trackio.alerts import AlertLevel
|
| 19 |
+
from trackio.api import Api
|
| 20 |
+
from trackio.apple_gpu import apple_gpu_available
|
| 21 |
+
from trackio.apple_gpu import log_apple_gpu as _log_apple_gpu
|
| 22 |
+
from trackio.deploy import freeze, sync
|
| 23 |
+
from trackio.gpu import gpu_available
|
| 24 |
+
from trackio.gpu import log_gpu as _log_nvidia_gpu
|
| 25 |
+
from trackio.histogram import Histogram
|
| 26 |
+
from trackio.imports import import_csv, import_tf_events
|
| 27 |
+
from trackio.launch import launch_trackio_dashboard
|
| 28 |
+
from trackio.markdown import Markdown
|
| 29 |
+
from trackio.media import (
|
| 30 |
+
TrackioAudio,
|
| 31 |
+
TrackioImage,
|
| 32 |
+
TrackioVideo,
|
| 33 |
+
get_project_media_path,
|
| 34 |
+
)
|
| 35 |
+
from trackio.run import Run
|
| 36 |
+
from trackio.server import TrackioDashboardApp, build_starlette_app_only
|
| 37 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 38 |
+
from trackio.table import Table
|
| 39 |
+
from trackio.typehints import UploadEntry
|
| 40 |
+
from trackio.utils import TRACKIO_DIR
|
| 41 |
+
|
| 42 |
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
| 43 |
+
|
| 44 |
+
__version__ = json.loads(Path(__file__).parent.joinpath("package.json").read_text())[
|
| 45 |
+
"version"
|
| 46 |
+
]
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class _TupleNoPrint(tuple):
|
| 50 |
+
def __repr__(self) -> str:
|
| 51 |
+
return ""
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
__all__ = [
|
| 55 |
+
"init",
|
| 56 |
+
"log",
|
| 57 |
+
"log_system",
|
| 58 |
+
"log_gpu",
|
| 59 |
+
"finish",
|
| 60 |
+
"alert",
|
| 61 |
+
"AlertLevel",
|
| 62 |
+
"show",
|
| 63 |
+
"sync",
|
| 64 |
+
"freeze",
|
| 65 |
+
"delete_project",
|
| 66 |
+
"import_csv",
|
| 67 |
+
"import_tf_events",
|
| 68 |
+
"save",
|
| 69 |
+
"Image",
|
| 70 |
+
"Video",
|
| 71 |
+
"Audio",
|
| 72 |
+
"Table",
|
| 73 |
+
"Histogram",
|
| 74 |
+
"Markdown",
|
| 75 |
+
"Api",
|
| 76 |
+
]
|
| 77 |
+
|
| 78 |
+
Audio = TrackioAudio
|
| 79 |
+
Image = TrackioImage
|
| 80 |
+
Video = TrackioVideo
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
config = {}
|
| 84 |
+
|
| 85 |
+
_atexit_registered = False
|
| 86 |
+
_projects_notified_auto_log_hw: set[str] = set()
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _cleanup_current_run():
|
| 90 |
+
run = context_vars.current_run.get()
|
| 91 |
+
if run is not None:
|
| 92 |
+
try:
|
| 93 |
+
run.finish()
|
| 94 |
+
except Exception:
|
| 95 |
+
pass
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def init(
|
| 99 |
+
project: str,
|
| 100 |
+
name: str | None = None,
|
| 101 |
+
group: str | None = None,
|
| 102 |
+
space_id: str | None = None,
|
| 103 |
+
space_storage: SpaceStorage | None = None,
|
| 104 |
+
dataset_id: str | None = None,
|
| 105 |
+
bucket_id: str | None = None,
|
| 106 |
+
config: dict | None = None,
|
| 107 |
+
resume: str = "never",
|
| 108 |
+
settings: Any = None,
|
| 109 |
+
private: bool | None = None,
|
| 110 |
+
embed: bool = True,
|
| 111 |
+
auto_log_gpu: bool | None = None,
|
| 112 |
+
gpu_log_interval: float = 10.0,
|
| 113 |
+
webhook_url: str | None = None,
|
| 114 |
+
webhook_min_level: AlertLevel | str | None = None,
|
| 115 |
+
) -> Run:
|
| 116 |
+
"""
|
| 117 |
+
Creates a new Trackio project and returns a [`Run`] object.
|
| 118 |
+
|
| 119 |
+
Args:
|
| 120 |
+
project (`str`):
|
| 121 |
+
The name of the project (can be an existing project to continue tracking or
|
| 122 |
+
a new project to start tracking from scratch).
|
| 123 |
+
name (`str`, *optional*):
|
| 124 |
+
The name of the run (if not provided, a default name will be generated).
|
| 125 |
+
group (`str`, *optional*):
|
| 126 |
+
The name of the group which this run belongs to in order to help organize
|
| 127 |
+
related runs together. You can toggle the entire group's visibilitiy in the
|
| 128 |
+
dashboard.
|
| 129 |
+
space_id (`str`, *optional*):
|
| 130 |
+
If provided, the project will be logged to a Hugging Face Space instead of
|
| 131 |
+
a local directory. Should be a complete Space name like
|
| 132 |
+
`"username/reponame"` or `"orgname/reponame"`, or just `"reponame"` in which
|
| 133 |
+
case the Space will be created in the currently-logged-in Hugging Face
|
| 134 |
+
user's namespace. If the Space does not exist, it will be created. If the
|
| 135 |
+
Space already exists, the project will be logged to it. Can also be set
|
| 136 |
+
via the `TRACKIO_SPACE_ID` environment variable. You cannot log to a
|
| 137 |
+
Space that has been **frozen** (converted to the static SDK); use
|
| 138 |
+
``trackio.sync(..., sdk="static")`` only after you are done logging.
|
| 139 |
+
space_storage ([`~huggingface_hub.SpaceStorage`], *optional*):
|
| 140 |
+
Choice of persistent storage tier.
|
| 141 |
+
dataset_id (`str`, *optional*):
|
| 142 |
+
Deprecated. Use `bucket_id` instead.
|
| 143 |
+
bucket_id (`str`, *optional*):
|
| 144 |
+
The ID of the Hugging Face Bucket to use for metric persistence. By default,
|
| 145 |
+
when a `space_id` is provided and `bucket_id` is not explicitly set, a
|
| 146 |
+
bucket is auto-generated from the space_id. Buckets provide
|
| 147 |
+
S3-like storage without git overhead - the SQLite database is stored directly
|
| 148 |
+
via `hf-mount` in the Space. Specify a Bucket with name like
|
| 149 |
+
`"username/bucketname"` or just `"bucketname"`.
|
| 150 |
+
config (`dict`, *optional*):
|
| 151 |
+
A dictionary of configuration options. Provided for compatibility with
|
| 152 |
+
`wandb.init()`.
|
| 153 |
+
resume (`str`, *optional*, defaults to `"never"`):
|
| 154 |
+
Controls how to handle resuming a run. Can be one of:
|
| 155 |
+
|
| 156 |
+
- `"must"`: Must resume the run with the given name, raises error if run
|
| 157 |
+
doesn't exist
|
| 158 |
+
- `"allow"`: Resume the run if it exists, otherwise create a new run
|
| 159 |
+
- `"never"`: Never resume a run, always create a new one
|
| 160 |
+
private (`bool`, *optional*):
|
| 161 |
+
Whether to make the Space private. If None (default), the repo will be
|
| 162 |
+
public unless the organization's default is private. This value is ignored
|
| 163 |
+
if the repo already exists.
|
| 164 |
+
settings (`Any`, *optional*):
|
| 165 |
+
Not used. Provided for compatibility with `wandb.init()`.
|
| 166 |
+
embed (`bool`, *optional*, defaults to `True`):
|
| 167 |
+
If running inside a Jupyter/Colab notebook, whether the dashboard should
|
| 168 |
+
automatically be embedded in the cell when trackio.init() is called. For
|
| 169 |
+
local runs, this launches a local Gradio app and embeds it. For Space runs,
|
| 170 |
+
this embeds the Space URL. In Colab, the local dashboard will be accessible
|
| 171 |
+
via a public share URL (default Gradio behavior).
|
| 172 |
+
auto_log_gpu (`bool` or `None`, *optional*, defaults to `None`):
|
| 173 |
+
Controls automatic GPU metrics logging. If `None` (default), GPU logging
|
| 174 |
+
is automatically enabled when `nvidia-ml-py` is installed and an NVIDIA
|
| 175 |
+
GPU or Apple M series is detected. Set to `True` to force enable or
|
| 176 |
+
`False` to disable.
|
| 177 |
+
gpu_log_interval (`float`, *optional*, defaults to `10.0`):
|
| 178 |
+
The interval in seconds between automatic GPU metric logs.
|
| 179 |
+
Only used when `auto_log_gpu=True`.
|
| 180 |
+
webhook_url (`str`, *optional*):
|
| 181 |
+
A webhook URL to POST alert payloads to when `trackio.alert()` is
|
| 182 |
+
called. Supports Slack and Discord webhook URLs natively (payloads
|
| 183 |
+
are formatted automatically). Can also be set via the
|
| 184 |
+
`TRACKIO_WEBHOOK_URL` environment variable. Individual alerts can
|
| 185 |
+
override this URL by passing `webhook_url` to `trackio.alert()`.
|
| 186 |
+
webhook_min_level (`AlertLevel` or `str`, *optional*):
|
| 187 |
+
Minimum alert level that should trigger webhook delivery.
|
| 188 |
+
For example, `AlertLevel.WARN` sends only `WARN` and `ERROR`
|
| 189 |
+
alerts to the webhook destination. Can also be set via
|
| 190 |
+
`TRACKIO_WEBHOOK_MIN_LEVEL`.
|
| 191 |
+
Returns:
|
| 192 |
+
`Run`: A [`Run`] object that can be used to log metrics and finish the run.
|
| 193 |
+
"""
|
| 194 |
+
if settings is not None:
|
| 195 |
+
warnings.warn(
|
| 196 |
+
"* Warning: settings is not used. Provided for compatibility with wandb.init(). Please create an issue at: https://github.com/gradio-app/trackio/issues if you need a specific feature implemented."
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
space_id = space_id or os.environ.get("TRACKIO_SPACE_ID")
|
| 200 |
+
bucket_id = bucket_id or os.environ.get("TRACKIO_BUCKET_ID")
|
| 201 |
+
if space_id is None and dataset_id is not None:
|
| 202 |
+
raise ValueError("Must provide a `space_id` when `dataset_id` is provided.")
|
| 203 |
+
if dataset_id is not None and bucket_id is not None:
|
| 204 |
+
raise ValueError("Cannot provide both `dataset_id` and `bucket_id`.")
|
| 205 |
+
try:
|
| 206 |
+
space_id, dataset_id, bucket_id = utils.preprocess_space_and_dataset_ids(
|
| 207 |
+
space_id, dataset_id, bucket_id
|
| 208 |
+
)
|
| 209 |
+
except LocalTokenNotFoundError as e:
|
| 210 |
+
raise LocalTokenNotFoundError(
|
| 211 |
+
f"You must be logged in to Hugging Face locally when `space_id` is provided to deploy to a Space. {e}"
|
| 212 |
+
) from e
|
| 213 |
+
|
| 214 |
+
if space_id is None and bucket_id is not None:
|
| 215 |
+
warnings.warn(
|
| 216 |
+
"trackio.init() has `bucket_id` set but `space_id` is None: metrics will be logged "
|
| 217 |
+
"locally only. Pass `space_id` to create or use a Hugging Face Space, which will be "
|
| 218 |
+
"attached to the Hugging Face Bucket.",
|
| 219 |
+
UserWarning,
|
| 220 |
+
stacklevel=2,
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
if space_id is not None:
|
| 224 |
+
deploy.raise_if_space_is_frozen_for_logging(space_id)
|
| 225 |
+
|
| 226 |
+
url = context_vars.current_server.get()
|
| 227 |
+
|
| 228 |
+
if space_id is not None:
|
| 229 |
+
if url is None:
|
| 230 |
+
url = space_id
|
| 231 |
+
context_vars.current_server.set(url)
|
| 232 |
+
context_vars.current_space_id.set(space_id)
|
| 233 |
+
|
| 234 |
+
_should_embed_local = False
|
| 235 |
+
|
| 236 |
+
if (
|
| 237 |
+
context_vars.current_project.get() is None
|
| 238 |
+
or context_vars.current_project.get() != project
|
| 239 |
+
):
|
| 240 |
+
print(f"* Trackio project initialized: {project}")
|
| 241 |
+
|
| 242 |
+
if bucket_id is not None:
|
| 243 |
+
os.environ["TRACKIO_BUCKET_ID"] = bucket_id
|
| 244 |
+
bucket_url = f"https://huggingface.co/buckets/{bucket_id}"
|
| 245 |
+
print(
|
| 246 |
+
f"* Trackio metrics will be synced to Hugging Face Bucket: {bucket_url}"
|
| 247 |
+
)
|
| 248 |
+
elif dataset_id is not None:
|
| 249 |
+
os.environ["TRACKIO_DATASET_ID"] = dataset_id
|
| 250 |
+
print(
|
| 251 |
+
f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}"
|
| 252 |
+
)
|
| 253 |
+
if space_id is None:
|
| 254 |
+
print(f"* Trackio metrics logged to: {TRACKIO_DIR}")
|
| 255 |
+
_should_embed_local = embed and utils.is_in_notebook()
|
| 256 |
+
if not _should_embed_local:
|
| 257 |
+
utils.print_dashboard_instructions(project)
|
| 258 |
+
else:
|
| 259 |
+
deploy.create_space_if_not_exists(
|
| 260 |
+
space_id,
|
| 261 |
+
space_storage,
|
| 262 |
+
dataset_id,
|
| 263 |
+
bucket_id,
|
| 264 |
+
private,
|
| 265 |
+
)
|
| 266 |
+
user_name, space_name = space_id.split("/")
|
| 267 |
+
space_url = deploy.SPACE_HOST_URL.format(
|
| 268 |
+
user_name=user_name, space_name=space_name
|
| 269 |
+
)
|
| 270 |
+
if utils.is_in_notebook() and embed:
|
| 271 |
+
utils.embed_url_in_notebook(space_url)
|
| 272 |
+
context_vars.current_project.set(project)
|
| 273 |
+
|
| 274 |
+
if resume == "must":
|
| 275 |
+
if name is None:
|
| 276 |
+
raise ValueError("Must provide a run name when resume='must'")
|
| 277 |
+
if name not in SQLiteStorage.get_runs(project):
|
| 278 |
+
raise ValueError(f"Run '{name}' does not exist in project '{project}'")
|
| 279 |
+
resumed = True
|
| 280 |
+
elif resume == "allow":
|
| 281 |
+
resumed = name is not None and name in SQLiteStorage.get_runs(project)
|
| 282 |
+
elif resume == "never":
|
| 283 |
+
if name is not None and name in SQLiteStorage.get_runs(project):
|
| 284 |
+
warnings.warn(
|
| 285 |
+
f"* Warning: resume='never' but a run '{name}' already exists in "
|
| 286 |
+
f"project '{project}'. Generating a new name and instead. If you want "
|
| 287 |
+
"to resume this run, call init() with resume='must' or resume='allow'."
|
| 288 |
+
)
|
| 289 |
+
name = None
|
| 290 |
+
resumed = False
|
| 291 |
+
else:
|
| 292 |
+
raise ValueError("resume must be one of: 'must', 'allow', or 'never'")
|
| 293 |
+
|
| 294 |
+
if auto_log_gpu is None:
|
| 295 |
+
nvidia_available = gpu_available()
|
| 296 |
+
apple_available = apple_gpu_available()
|
| 297 |
+
auto_log_gpu = nvidia_available or apple_available
|
| 298 |
+
if project not in _projects_notified_auto_log_hw:
|
| 299 |
+
if nvidia_available:
|
| 300 |
+
print("* NVIDIA GPU detected, enabling automatic GPU metrics logging")
|
| 301 |
+
elif apple_available:
|
| 302 |
+
print(
|
| 303 |
+
"* Apple Silicon detected, enabling automatic system metrics logging"
|
| 304 |
+
)
|
| 305 |
+
if nvidia_available or apple_available:
|
| 306 |
+
_projects_notified_auto_log_hw.add(project)
|
| 307 |
+
|
| 308 |
+
run = Run(
|
| 309 |
+
url=url,
|
| 310 |
+
project=project,
|
| 311 |
+
client=None,
|
| 312 |
+
name=name,
|
| 313 |
+
group=group,
|
| 314 |
+
config=config,
|
| 315 |
+
space_id=space_id,
|
| 316 |
+
auto_log_gpu=auto_log_gpu,
|
| 317 |
+
gpu_log_interval=gpu_log_interval,
|
| 318 |
+
webhook_url=webhook_url,
|
| 319 |
+
webhook_min_level=webhook_min_level,
|
| 320 |
+
)
|
| 321 |
+
|
| 322 |
+
if space_id is not None:
|
| 323 |
+
SQLiteStorage.set_project_metadata(project, "space_id", space_id)
|
| 324 |
+
if SQLiteStorage.has_pending_data(project):
|
| 325 |
+
run._has_local_buffer = True
|
| 326 |
+
|
| 327 |
+
global _atexit_registered
|
| 328 |
+
if not _atexit_registered:
|
| 329 |
+
atexit.register(_cleanup_current_run)
|
| 330 |
+
_atexit_registered = True
|
| 331 |
+
|
| 332 |
+
if resumed:
|
| 333 |
+
print(f"* Resumed existing run: {run.name}")
|
| 334 |
+
else:
|
| 335 |
+
print(f"* Created new run: {run.name}")
|
| 336 |
+
|
| 337 |
+
context_vars.current_run.set(run)
|
| 338 |
+
globals()["config"] = run.config
|
| 339 |
+
|
| 340 |
+
if _should_embed_local:
|
| 341 |
+
show(project=project, open_browser=False, block_thread=False)
|
| 342 |
+
|
| 343 |
+
return run
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
def log(metrics: dict, step: int | None = None) -> None:
|
| 347 |
+
"""
|
| 348 |
+
Logs metrics to the current run.
|
| 349 |
+
|
| 350 |
+
Args:
|
| 351 |
+
metrics (`dict`):
|
| 352 |
+
A dictionary of metrics to log.
|
| 353 |
+
step (`int`, *optional*):
|
| 354 |
+
The step number. If not provided, the step will be incremented
|
| 355 |
+
automatically.
|
| 356 |
+
"""
|
| 357 |
+
run = context_vars.current_run.get()
|
| 358 |
+
if run is None:
|
| 359 |
+
raise RuntimeError("Call trackio.init() before trackio.log().")
|
| 360 |
+
run.log(
|
| 361 |
+
metrics=metrics,
|
| 362 |
+
step=step,
|
| 363 |
+
)
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def log_system(metrics: dict) -> None:
|
| 367 |
+
"""
|
| 368 |
+
Logs system metrics (GPU, etc.) to the current run using timestamps instead of steps.
|
| 369 |
+
|
| 370 |
+
Args:
|
| 371 |
+
metrics (`dict`):
|
| 372 |
+
A dictionary of system metrics to log.
|
| 373 |
+
"""
|
| 374 |
+
run = context_vars.current_run.get()
|
| 375 |
+
if run is None:
|
| 376 |
+
raise RuntimeError("Call trackio.init() before trackio.log_system().")
|
| 377 |
+
run.log_system(metrics=metrics)
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
def log_gpu(run: Run | None = None, device: int | None = None) -> dict:
|
| 381 |
+
"""
|
| 382 |
+
Log GPU metrics to the current or specified run as system metrics.
|
| 383 |
+
Automatically detects whether an NVIDIA or Apple GPU is available and calls
|
| 384 |
+
the appropriate logging method.
|
| 385 |
+
|
| 386 |
+
Args:
|
| 387 |
+
run: Optional Run instance. If None, uses current run from context.
|
| 388 |
+
device: CUDA device index to collect metrics from (NVIDIA GPUs only).
|
| 389 |
+
If None, collects from all GPUs visible to this process.
|
| 390 |
+
This parameter is ignored for Apple GPUs.
|
| 391 |
+
|
| 392 |
+
Returns:
|
| 393 |
+
dict: The GPU metrics that were logged.
|
| 394 |
+
|
| 395 |
+
Example:
|
| 396 |
+
```python
|
| 397 |
+
import trackio
|
| 398 |
+
|
| 399 |
+
run = trackio.init(project="my-project")
|
| 400 |
+
trackio.log({"loss": 0.5})
|
| 401 |
+
trackio.log_gpu()
|
| 402 |
+
trackio.log_gpu(device=0)
|
| 403 |
+
```
|
| 404 |
+
"""
|
| 405 |
+
if run is None:
|
| 406 |
+
run = context_vars.current_run.get()
|
| 407 |
+
if run is None:
|
| 408 |
+
raise RuntimeError("Call trackio.init() before trackio.log_gpu().")
|
| 409 |
+
|
| 410 |
+
if gpu_available():
|
| 411 |
+
return _log_nvidia_gpu(run=run, device=device)
|
| 412 |
+
elif apple_gpu_available():
|
| 413 |
+
return _log_apple_gpu(run=run)
|
| 414 |
+
else:
|
| 415 |
+
warnings.warn(
|
| 416 |
+
"No GPU detected. Install nvidia-ml-py for NVIDIA GPU support "
|
| 417 |
+
"or psutil for Apple Silicon support."
|
| 418 |
+
)
|
| 419 |
+
return {}
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
def finish():
|
| 423 |
+
"""
|
| 424 |
+
Finishes the current run.
|
| 425 |
+
"""
|
| 426 |
+
run = context_vars.current_run.get()
|
| 427 |
+
if run is None:
|
| 428 |
+
raise RuntimeError("Call trackio.init() before trackio.finish().")
|
| 429 |
+
run.finish()
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
def alert(
|
| 433 |
+
title: str,
|
| 434 |
+
text: str | None = None,
|
| 435 |
+
level: AlertLevel = AlertLevel.WARN,
|
| 436 |
+
webhook_url: str | None = None,
|
| 437 |
+
) -> None:
|
| 438 |
+
"""
|
| 439 |
+
Fires an alert immediately on the current run. The alert is printed to the
|
| 440 |
+
terminal, stored in the database, and displayed in the dashboard. If a
|
| 441 |
+
webhook URL is configured (via `trackio.init()`, the `TRACKIO_WEBHOOK_URL`
|
| 442 |
+
environment variable, or the `webhook_url` parameter here), the alert is
|
| 443 |
+
also POSTed to that URL.
|
| 444 |
+
|
| 445 |
+
Args:
|
| 446 |
+
title (`str`):
|
| 447 |
+
A short title for the alert.
|
| 448 |
+
text (`str`, *optional*):
|
| 449 |
+
A longer description with details about the alert.
|
| 450 |
+
level (`AlertLevel`, *optional*, defaults to `AlertLevel.WARN`):
|
| 451 |
+
The severity level. One of `AlertLevel.INFO`, `AlertLevel.WARN`,
|
| 452 |
+
or `AlertLevel.ERROR`.
|
| 453 |
+
webhook_url (`str`, *optional*):
|
| 454 |
+
A webhook URL to send this specific alert to. Overrides any
|
| 455 |
+
URL set in `trackio.init()` or the `TRACKIO_WEBHOOK_URL`
|
| 456 |
+
environment variable. Supports Slack and Discord webhook
|
| 457 |
+
URLs natively.
|
| 458 |
+
"""
|
| 459 |
+
run = context_vars.current_run.get()
|
| 460 |
+
if run is None:
|
| 461 |
+
raise RuntimeError("Call trackio.init() before trackio.alert().")
|
| 462 |
+
run.alert(title=title, text=text, level=level, webhook_url=webhook_url)
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
def delete_project(project: str, force: bool = False) -> bool:
|
| 466 |
+
"""
|
| 467 |
+
Deletes a project by removing its local SQLite database.
|
| 468 |
+
|
| 469 |
+
Args:
|
| 470 |
+
project (`str`):
|
| 471 |
+
The name of the project to delete.
|
| 472 |
+
force (`bool`, *optional*, defaults to `False`):
|
| 473 |
+
If `True`, deletes the project without prompting for confirmation.
|
| 474 |
+
If `False`, prompts the user to confirm before deleting.
|
| 475 |
+
|
| 476 |
+
Returns:
|
| 477 |
+
`bool`: `True` if the project was deleted, `False` otherwise.
|
| 478 |
+
"""
|
| 479 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 480 |
+
|
| 481 |
+
if not db_path.exists():
|
| 482 |
+
print(f"* Project '{project}' does not exist.")
|
| 483 |
+
return False
|
| 484 |
+
|
| 485 |
+
if not force:
|
| 486 |
+
response = input(
|
| 487 |
+
f"Are you sure you want to delete project '{project}'? "
|
| 488 |
+
f"This will permanently delete all runs and metrics. (y/N): "
|
| 489 |
+
)
|
| 490 |
+
if response.lower() not in ["y", "yes"]:
|
| 491 |
+
print("* Deletion cancelled.")
|
| 492 |
+
return False
|
| 493 |
+
|
| 494 |
+
try:
|
| 495 |
+
db_path.unlink()
|
| 496 |
+
|
| 497 |
+
for suffix in ("-wal", "-shm"):
|
| 498 |
+
sidecar = Path(str(db_path) + suffix)
|
| 499 |
+
if sidecar.exists():
|
| 500 |
+
sidecar.unlink()
|
| 501 |
+
|
| 502 |
+
print(f"* Project '{project}' has been deleted.")
|
| 503 |
+
return True
|
| 504 |
+
except Exception as e:
|
| 505 |
+
print(f"* Error deleting project '{project}': {e}")
|
| 506 |
+
return False
|
| 507 |
+
|
| 508 |
+
|
| 509 |
+
def save(
|
| 510 |
+
glob_str: str | Path,
|
| 511 |
+
project: str | None = None,
|
| 512 |
+
) -> str:
|
| 513 |
+
"""
|
| 514 |
+
Saves files to a project (not linked to a specific run). If Trackio is running
|
| 515 |
+
locally, the file(s) will be copied to the project's files directory. If Trackio is
|
| 516 |
+
running in a Space, the file(s) will be uploaded to the Space's files directory.
|
| 517 |
+
|
| 518 |
+
Args:
|
| 519 |
+
glob_str (`str` or `Path`):
|
| 520 |
+
The file path or glob pattern to save. Can be a single file or a pattern
|
| 521 |
+
matching multiple files (e.g., `"*.py"`, `"models/**/*.pth"`).
|
| 522 |
+
project (`str`, *optional*):
|
| 523 |
+
The name of the project to save files to. If not provided, uses the current
|
| 524 |
+
project from `trackio.init()`. If no project is initialized, raises an
|
| 525 |
+
error.
|
| 526 |
+
|
| 527 |
+
Returns:
|
| 528 |
+
`str`: The path where the file(s) were saved (project's files directory).
|
| 529 |
+
|
| 530 |
+
Example:
|
| 531 |
+
```python
|
| 532 |
+
import trackio
|
| 533 |
+
|
| 534 |
+
trackio.init(project="my-project")
|
| 535 |
+
trackio.save("config.yaml")
|
| 536 |
+
trackio.save("models/*.pth")
|
| 537 |
+
```
|
| 538 |
+
"""
|
| 539 |
+
if project is None:
|
| 540 |
+
project = context_vars.current_project.get()
|
| 541 |
+
if project is None:
|
| 542 |
+
raise RuntimeError(
|
| 543 |
+
"No project specified. Either call trackio.init() first or provide a "
|
| 544 |
+
"project parameter to trackio.save()."
|
| 545 |
+
)
|
| 546 |
+
|
| 547 |
+
glob_str = Path(glob_str)
|
| 548 |
+
base_path = Path.cwd().resolve()
|
| 549 |
+
|
| 550 |
+
matched_files = []
|
| 551 |
+
if glob_str.is_file():
|
| 552 |
+
matched_files = [glob_str.resolve()]
|
| 553 |
+
else:
|
| 554 |
+
pattern = str(glob_str)
|
| 555 |
+
if not glob_str.is_absolute():
|
| 556 |
+
pattern = str((Path.cwd() / glob_str).resolve())
|
| 557 |
+
matched_files = [
|
| 558 |
+
Path(f).resolve()
|
| 559 |
+
for f in glob.glob(pattern, recursive=True)
|
| 560 |
+
if Path(f).is_file()
|
| 561 |
+
]
|
| 562 |
+
|
| 563 |
+
if not matched_files:
|
| 564 |
+
raise ValueError(f"No files found matching pattern: {glob_str}")
|
| 565 |
+
|
| 566 |
+
current_run = context_vars.current_run.get()
|
| 567 |
+
is_local = (
|
| 568 |
+
current_run._is_local
|
| 569 |
+
if current_run is not None
|
| 570 |
+
else (context_vars.current_space_id.get() is None)
|
| 571 |
+
)
|
| 572 |
+
|
| 573 |
+
if is_local:
|
| 574 |
+
for file_path in matched_files:
|
| 575 |
+
try:
|
| 576 |
+
relative_to_base = file_path.relative_to(base_path)
|
| 577 |
+
except ValueError:
|
| 578 |
+
relative_to_base = Path(file_path.name)
|
| 579 |
+
|
| 580 |
+
if current_run is not None:
|
| 581 |
+
current_run._queue_upload(
|
| 582 |
+
file_path,
|
| 583 |
+
step=None,
|
| 584 |
+
relative_path=str(relative_to_base.parent),
|
| 585 |
+
use_run_name=False,
|
| 586 |
+
)
|
| 587 |
+
else:
|
| 588 |
+
media_path = get_project_media_path(
|
| 589 |
+
project=project,
|
| 590 |
+
run=None,
|
| 591 |
+
step=None,
|
| 592 |
+
relative_path=str(relative_to_base),
|
| 593 |
+
)
|
| 594 |
+
shutil.copy(str(file_path), str(media_path))
|
| 595 |
+
else:
|
| 596 |
+
url = context_vars.current_server.get()
|
| 597 |
+
|
| 598 |
+
upload_entries = []
|
| 599 |
+
for file_path in matched_files:
|
| 600 |
+
try:
|
| 601 |
+
relative_to_base = file_path.relative_to(base_path)
|
| 602 |
+
except ValueError:
|
| 603 |
+
relative_to_base = Path(file_path.name)
|
| 604 |
+
|
| 605 |
+
if current_run is not None:
|
| 606 |
+
current_run._queue_upload(
|
| 607 |
+
file_path,
|
| 608 |
+
step=None,
|
| 609 |
+
relative_path=str(relative_to_base.parent),
|
| 610 |
+
use_run_name=False,
|
| 611 |
+
)
|
| 612 |
+
else:
|
| 613 |
+
upload_entry: UploadEntry = {
|
| 614 |
+
"project": project,
|
| 615 |
+
"run": None,
|
| 616 |
+
"step": None,
|
| 617 |
+
"relative_path": str(relative_to_base),
|
| 618 |
+
"uploaded_file": handle_file(file_path),
|
| 619 |
+
}
|
| 620 |
+
upload_entries.append(upload_entry)
|
| 621 |
+
|
| 622 |
+
if upload_entries:
|
| 623 |
+
if url is None:
|
| 624 |
+
raise RuntimeError(
|
| 625 |
+
"No server available. Call trackio.init() before trackio.save() to start the server."
|
| 626 |
+
)
|
| 627 |
+
|
| 628 |
+
try:
|
| 629 |
+
client = Client(url, verbose=False, httpx_kwargs={"timeout": 90})
|
| 630 |
+
client.predict(
|
| 631 |
+
api_name="/bulk_upload_media",
|
| 632 |
+
uploads=upload_entries,
|
| 633 |
+
hf_token=huggingface_hub.utils.get_token(),
|
| 634 |
+
)
|
| 635 |
+
except Exception as e:
|
| 636 |
+
warnings.warn(
|
| 637 |
+
f"Failed to upload files: {e}. "
|
| 638 |
+
"Files may not be available in the dashboard."
|
| 639 |
+
)
|
| 640 |
+
|
| 641 |
+
return str(utils.MEDIA_DIR / project / "files")
|
| 642 |
+
|
| 643 |
+
|
| 644 |
+
def show(
|
| 645 |
+
project: str | None = None,
|
| 646 |
+
*,
|
| 647 |
+
theme: Any = None,
|
| 648 |
+
mcp_server: bool | None = None,
|
| 649 |
+
footer: bool = True,
|
| 650 |
+
color_palette: list[str] | None = None,
|
| 651 |
+
open_browser: bool = True,
|
| 652 |
+
block_thread: bool | None = None,
|
| 653 |
+
host: str | None = None,
|
| 654 |
+
share: bool | None = None,
|
| 655 |
+
server_port: int | None = None,
|
| 656 |
+
):
|
| 657 |
+
"""
|
| 658 |
+
Launches the Trackio dashboard.
|
| 659 |
+
|
| 660 |
+
Args:
|
| 661 |
+
project (`str`, *optional*):
|
| 662 |
+
The name of the project whose runs to show. If not provided, all projects
|
| 663 |
+
will be shown and the user can select one.
|
| 664 |
+
theme (`Any`, *optional*):
|
| 665 |
+
Ignored. Kept for backward compatibility; Trackio no longer uses Gradio themes.
|
| 666 |
+
mcp_server (`bool`, *optional*):
|
| 667 |
+
If `True`, the dashboard enables MCP lifespan hooks when available. If `None`
|
| 668 |
+
(default), the `GRADIO_MCP_SERVER` environment variable is used (e.g. on Spaces).
|
| 669 |
+
footer (`bool`, *optional*, defaults to `True`):
|
| 670 |
+
Whether to include `footer=false` in the write-token URL when `False`.
|
| 671 |
+
This can also be controlled via the `footer` query parameter in the URL.
|
| 672 |
+
color_palette (`list[str]`, *optional*):
|
| 673 |
+
A list of hex color codes to use for plot lines. If not provided, the
|
| 674 |
+
`TRACKIO_COLOR_PALETTE` environment variable will be used (comma-separated
|
| 675 |
+
hex codes), or if that is not set, the default color palette will be used.
|
| 676 |
+
Example: `['#FF0000', '#00FF00', '#0000FF']`
|
| 677 |
+
open_browser (`bool`, *optional*, defaults to `True`):
|
| 678 |
+
If `True` and not in a notebook, a new browser tab will be opened with the
|
| 679 |
+
dashboard. If `False`, the browser will not be opened.
|
| 680 |
+
block_thread (`bool`, *optional*):
|
| 681 |
+
If `True`, the main thread will be blocked until the dashboard is closed.
|
| 682 |
+
If `None` (default behavior), then the main thread will not be blocked if the
|
| 683 |
+
dashboard is launched in a notebook, otherwise the main thread will be blocked.
|
| 684 |
+
host (`str`, *optional*):
|
| 685 |
+
The host to bind the server to. If not provided, defaults to `'127.0.0.1'`
|
| 686 |
+
(localhost only). Set to `'0.0.0.0'` to allow remote access.
|
| 687 |
+
share (`bool`, *optional*):
|
| 688 |
+
If `True`, creates a temporary public URL (Gradio-compatible tunnel). On Colab
|
| 689 |
+
or hosted notebooks, defaults to `True` unless overridden.
|
| 690 |
+
server_port (`int`, *optional*):
|
| 691 |
+
Port to bind. If not set, scans from `GRADIO_SERVER_PORT` (default 7860).
|
| 692 |
+
|
| 693 |
+
Returns:
|
| 694 |
+
`app`: The dashboard handle (`.close()` stops the server).
|
| 695 |
+
`url`: The local URL of the dashboard.
|
| 696 |
+
`share_url`: The public share URL, if any.
|
| 697 |
+
`full_url`: The full URL including the write token (share URL when sharing, else local).
|
| 698 |
+
"""
|
| 699 |
+
if theme is not None:
|
| 700 |
+
warnings.warn(
|
| 701 |
+
"The theme argument is ignored; Trackio no longer depends on Gradio themes.",
|
| 702 |
+
UserWarning,
|
| 703 |
+
stacklevel=2,
|
| 704 |
+
)
|
| 705 |
+
|
| 706 |
+
if color_palette is not None:
|
| 707 |
+
os.environ["TRACKIO_COLOR_PALETTE"] = ",".join(color_palette)
|
| 708 |
+
|
| 709 |
+
_mcp_server = (
|
| 710 |
+
mcp_server
|
| 711 |
+
if mcp_server is not None
|
| 712 |
+
else os.environ.get("GRADIO_MCP_SERVER", "False") == "True"
|
| 713 |
+
)
|
| 714 |
+
|
| 715 |
+
starlette_app, wt = build_starlette_app_only(mcp_server=_mcp_server)
|
| 716 |
+
local_url, share_url, _local_api_url, uv_server = launch_trackio_dashboard(
|
| 717 |
+
starlette_app,
|
| 718 |
+
server_name=host,
|
| 719 |
+
server_port=server_port,
|
| 720 |
+
share=share,
|
| 721 |
+
mcp_server=_mcp_server,
|
| 722 |
+
quiet=True,
|
| 723 |
+
)
|
| 724 |
+
server = TrackioDashboardApp(starlette_app, uv_server, wt)
|
| 725 |
+
|
| 726 |
+
base_root = (share_url or local_url).rstrip("/")
|
| 727 |
+
base_url = base_root + "/"
|
| 728 |
+
dashboard_url = base_url
|
| 729 |
+
if project:
|
| 730 |
+
dashboard_url += f"?project={project}"
|
| 731 |
+
full_url = utils.get_full_url(
|
| 732 |
+
base_root,
|
| 733 |
+
project=project,
|
| 734 |
+
write_token=wt,
|
| 735 |
+
footer=footer,
|
| 736 |
+
)
|
| 737 |
+
|
| 738 |
+
if not utils.is_in_notebook():
|
| 739 |
+
print(f"* Trackio UI launched at: {dashboard_url}")
|
| 740 |
+
print(f"* API (gradio_client-compatible) at: {base_url}gradio_api/")
|
| 741 |
+
if open_browser:
|
| 742 |
+
webbrowser.open(dashboard_url)
|
| 743 |
+
block_thread = block_thread if block_thread is not None else True
|
| 744 |
+
else:
|
| 745 |
+
utils.embed_url_in_notebook(dashboard_url)
|
| 746 |
+
block_thread = block_thread if block_thread is not None else False
|
| 747 |
+
|
| 748 |
+
if block_thread:
|
| 749 |
+
utils.block_main_thread_until_keyboard_interrupt()
|
| 750 |
+
return _TupleNoPrint((server, local_url, share_url, full_url))
|
trackio/_vendor/__init__.py
ADDED
|
File without changes
|
trackio/_vendor/gradio_exceptions.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
class ChecksumMismatchError(Exception):
|
| 2 |
+
pass
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class ShareCertificateWriteError(Exception):
|
| 6 |
+
pass
|
trackio/_vendor/networking.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import time
|
| 5 |
+
import warnings
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from urllib.parse import urlparse, urlunparse
|
| 8 |
+
|
| 9 |
+
import httpx
|
| 10 |
+
|
| 11 |
+
from trackio._vendor.gradio_exceptions import ShareCertificateWriteError
|
| 12 |
+
from trackio._vendor.tunneling import CERTIFICATE_PATH, Tunnel
|
| 13 |
+
|
| 14 |
+
GRADIO_API_SERVER = "https://api.gradio.app/v3/tunnel-request"
|
| 15 |
+
GRADIO_SHARE_SERVER_ADDRESS = os.getenv("GRADIO_SHARE_SERVER_ADDRESS")
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def setup_tunnel(
|
| 19 |
+
local_host: str,
|
| 20 |
+
local_port: int,
|
| 21 |
+
share_token: str,
|
| 22 |
+
share_server_address: str | None,
|
| 23 |
+
share_server_tls_certificate: str | None,
|
| 24 |
+
) -> str:
|
| 25 |
+
share_server_address = (
|
| 26 |
+
GRADIO_SHARE_SERVER_ADDRESS
|
| 27 |
+
if share_server_address is None
|
| 28 |
+
else share_server_address
|
| 29 |
+
)
|
| 30 |
+
if share_server_address is None:
|
| 31 |
+
try:
|
| 32 |
+
response = httpx.get(GRADIO_API_SERVER, timeout=30)
|
| 33 |
+
payload = response.json()[0]
|
| 34 |
+
remote_host, remote_port = payload["host"], int(payload["port"])
|
| 35 |
+
certificate = payload["root_ca"]
|
| 36 |
+
except Exception as e:
|
| 37 |
+
raise RuntimeError(
|
| 38 |
+
"Could not get share link from Gradio API Server."
|
| 39 |
+
) from e
|
| 40 |
+
try:
|
| 41 |
+
Path(CERTIFICATE_PATH).parent.mkdir(parents=True, exist_ok=True)
|
| 42 |
+
with open(CERTIFICATE_PATH, "w") as f:
|
| 43 |
+
f.write(certificate)
|
| 44 |
+
except Exception as e:
|
| 45 |
+
raise ShareCertificateWriteError(
|
| 46 |
+
f"{e}. This can happen if the current working directory is read-only."
|
| 47 |
+
) from e
|
| 48 |
+
share_server_tls_certificate = CERTIFICATE_PATH
|
| 49 |
+
|
| 50 |
+
else:
|
| 51 |
+
remote_host, remote_port = share_server_address.split(":")
|
| 52 |
+
remote_port = int(remote_port)
|
| 53 |
+
tunnel = Tunnel(
|
| 54 |
+
remote_host,
|
| 55 |
+
remote_port,
|
| 56 |
+
local_host,
|
| 57 |
+
local_port,
|
| 58 |
+
share_token,
|
| 59 |
+
share_server_tls_certificate,
|
| 60 |
+
)
|
| 61 |
+
address = tunnel.start_tunnel()
|
| 62 |
+
return address
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def url_ok(url: str) -> bool:
|
| 66 |
+
try:
|
| 67 |
+
for _ in range(5):
|
| 68 |
+
with warnings.catch_warnings():
|
| 69 |
+
warnings.filterwarnings("ignore")
|
| 70 |
+
r = httpx.head(url, timeout=3, verify=False)
|
| 71 |
+
if r.status_code in (200, 401, 302, 303, 307):
|
| 72 |
+
return True
|
| 73 |
+
time.sleep(0.500)
|
| 74 |
+
except (ConnectionError, httpx.ConnectError, httpx.TimeoutException):
|
| 75 |
+
return False
|
| 76 |
+
return False
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def normalize_share_url(share_url: str, share_server_protocol: str) -> str:
|
| 80 |
+
parsed_url = urlparse(share_url)
|
| 81 |
+
return urlunparse((share_server_protocol,) + parsed_url[1:])
|
trackio/_vendor/tunneling.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import atexit
|
| 2 |
+
import hashlib
|
| 3 |
+
import os
|
| 4 |
+
import platform
|
| 5 |
+
import re
|
| 6 |
+
import stat
|
| 7 |
+
import subprocess
|
| 8 |
+
import sys
|
| 9 |
+
import time
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
import httpx
|
| 13 |
+
from huggingface_hub.constants import HF_HOME
|
| 14 |
+
|
| 15 |
+
from trackio._vendor.gradio_exceptions import ChecksumMismatchError
|
| 16 |
+
|
| 17 |
+
VERSION = "0.3"
|
| 18 |
+
CURRENT_TUNNELS: list["Tunnel"] = []
|
| 19 |
+
|
| 20 |
+
machine = platform.machine()
|
| 21 |
+
if machine == "x86_64":
|
| 22 |
+
machine = "amd64"
|
| 23 |
+
elif machine == "aarch64":
|
| 24 |
+
machine = "arm64"
|
| 25 |
+
|
| 26 |
+
BINARY_REMOTE_NAME = f"frpc_{platform.system().lower()}_{machine.lower()}"
|
| 27 |
+
EXTENSION = ".exe" if os.name == "nt" else ""
|
| 28 |
+
BINARY_URL = f"https://cdn-media.huggingface.co/frpc-gradio-{VERSION}/{BINARY_REMOTE_NAME}{EXTENSION}"
|
| 29 |
+
|
| 30 |
+
CHECKSUMS = {
|
| 31 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_windows_amd64.exe": "14bc0ea470be5d67d79a07412bd21de8a0a179c6ac1116d7764f68e942dc9ceb",
|
| 32 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_amd64": "c791d1f047b41ff5885772fc4bf20b797c6059bbd82abb9e31de15e55d6a57c4",
|
| 33 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_arm64": "823ced25104de6dc3c9f4798dbb43f20e681207279e6ab89c40e2176ccbf70cd",
|
| 34 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_darwin_amd64": "930f8face3365810ce16689da81b7d1941fda4466225a7bbcbced9a2916a6e15",
|
| 35 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_darwin_arm64": "dfac50c690aca459ed5158fad8bfbe99f9282baf4166cf7c410a6673fbc1f327",
|
| 36 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_arm": "4b563beb2e36c448cc688174e20b53af38dc1ff2b5e362d4ddd1401f2affbfb7",
|
| 37 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_freebsd_386": "cb0a56c764ecf96dd54ed601d240c564f060ee4e58202d65ffca17c1a51ce19c",
|
| 38 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_freebsd_amd64": "516d9e6903513869a011ddcd1ec206167ad1eb5dd6640d21057acc258edecbbb",
|
| 39 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_386": "4c2f2a48cd71571498c0ac8a4d42a055f22cb7f14b4b5a2b0d584220fd60a283",
|
| 40 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mips": "b309ecd594d4f0f7f33e556a80d4b67aef9319c00a8334648a618e56b23cb9e0",
|
| 41 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mips64": "0372ef5505baa6f3b64c6295a86541b24b7b0dbe4ef28b344992e21f47624b7b",
|
| 42 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_riscv64": "1658eed7e8c14ea76e1d95749d58441ce24147c3d559381832c725c29cfc3df3",
|
| 43 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mipsle": "a2aaba16961d3372b79bd7a28976fcd0f0bbaebc2b50d5a7a71af2240747960f",
|
| 44 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_windows_386.exe": "721b90550195a83e15f2176d8f85a48d5a25822757cb872e9723d4bccc4e5bb6",
|
| 45 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mips64le": "796481edd609f31962b45cc0ab4c9798d040205ae3bf354ed1b72fb432d796b8",
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
CHUNK_SIZE = 128
|
| 49 |
+
|
| 50 |
+
BINARY_FILENAME = f"{BINARY_REMOTE_NAME}_v{VERSION}"
|
| 51 |
+
BINARY_FOLDER = Path(HF_HOME) / "trackio" / "frpc"
|
| 52 |
+
BINARY_PATH = str(BINARY_FOLDER / BINARY_FILENAME)
|
| 53 |
+
|
| 54 |
+
TUNNEL_TIMEOUT_SECONDS = 30
|
| 55 |
+
TUNNEL_ERROR_MESSAGE = (
|
| 56 |
+
"Could not create share URL. "
|
| 57 |
+
"Please check the appended log from frpc for more information:"
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
CERTIFICATE_PATH = ".trackio/certificate.pem"
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class Tunnel:
|
| 64 |
+
def __init__(
|
| 65 |
+
self,
|
| 66 |
+
remote_host: str,
|
| 67 |
+
remote_port: int,
|
| 68 |
+
local_host: str,
|
| 69 |
+
local_port: int,
|
| 70 |
+
share_token: str,
|
| 71 |
+
share_server_tls_certificate: str | None,
|
| 72 |
+
):
|
| 73 |
+
self.proc = None
|
| 74 |
+
self.url = None
|
| 75 |
+
self.remote_host = remote_host
|
| 76 |
+
self.remote_port = remote_port
|
| 77 |
+
self.local_host = local_host
|
| 78 |
+
self.local_port = local_port
|
| 79 |
+
self.share_token = share_token
|
| 80 |
+
self.share_server_tls_certificate = share_server_tls_certificate
|
| 81 |
+
|
| 82 |
+
@staticmethod
|
| 83 |
+
def download_binary():
|
| 84 |
+
if not Path(BINARY_PATH).exists():
|
| 85 |
+
Path(BINARY_FOLDER).mkdir(parents=True, exist_ok=True)
|
| 86 |
+
resp = httpx.get(BINARY_URL, timeout=30)
|
| 87 |
+
|
| 88 |
+
if resp.status_code == 403:
|
| 89 |
+
raise OSError(
|
| 90 |
+
f"Cannot set up a share link as this platform is incompatible. Please "
|
| 91 |
+
f"create a GitHub issue with information about your platform: {platform.uname()}"
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
resp.raise_for_status()
|
| 95 |
+
|
| 96 |
+
with open(BINARY_PATH, "wb") as file:
|
| 97 |
+
file.write(resp.content)
|
| 98 |
+
st = os.stat(BINARY_PATH)
|
| 99 |
+
os.chmod(BINARY_PATH, st.st_mode | stat.S_IEXEC)
|
| 100 |
+
|
| 101 |
+
if BINARY_URL in CHECKSUMS:
|
| 102 |
+
sha = hashlib.sha256()
|
| 103 |
+
with open(BINARY_PATH, "rb") as f:
|
| 104 |
+
for chunk in iter(lambda: f.read(CHUNK_SIZE * sha.block_size), b""):
|
| 105 |
+
sha.update(chunk)
|
| 106 |
+
calculated_hash = sha.hexdigest()
|
| 107 |
+
|
| 108 |
+
if calculated_hash != CHECKSUMS[BINARY_URL]:
|
| 109 |
+
raise ChecksumMismatchError()
|
| 110 |
+
|
| 111 |
+
def start_tunnel(self) -> str:
|
| 112 |
+
self.download_binary()
|
| 113 |
+
self.url = self._start_tunnel(BINARY_PATH)
|
| 114 |
+
return self.url
|
| 115 |
+
|
| 116 |
+
def kill(self):
|
| 117 |
+
if self.proc is not None:
|
| 118 |
+
print(f"Killing tunnel {self.local_host}:{self.local_port} <> {self.url}")
|
| 119 |
+
self.proc.terminate()
|
| 120 |
+
self.proc = None
|
| 121 |
+
|
| 122 |
+
def _start_tunnel(self, binary: str) -> str:
|
| 123 |
+
CURRENT_TUNNELS.append(self)
|
| 124 |
+
command = [
|
| 125 |
+
binary,
|
| 126 |
+
"http",
|
| 127 |
+
"-n",
|
| 128 |
+
self.share_token,
|
| 129 |
+
"-l",
|
| 130 |
+
str(self.local_port),
|
| 131 |
+
"-i",
|
| 132 |
+
self.local_host,
|
| 133 |
+
"--uc",
|
| 134 |
+
"--sd",
|
| 135 |
+
"random",
|
| 136 |
+
"--ue",
|
| 137 |
+
"--server_addr",
|
| 138 |
+
f"{self.remote_host}:{self.remote_port}",
|
| 139 |
+
"--disable_log_color",
|
| 140 |
+
]
|
| 141 |
+
if self.share_server_tls_certificate is not None:
|
| 142 |
+
command.extend(
|
| 143 |
+
[
|
| 144 |
+
"--tls_enable",
|
| 145 |
+
"--tls_trusted_ca_file",
|
| 146 |
+
self.share_server_tls_certificate,
|
| 147 |
+
]
|
| 148 |
+
)
|
| 149 |
+
self.proc = subprocess.Popen(
|
| 150 |
+
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
| 151 |
+
)
|
| 152 |
+
atexit.register(self.kill)
|
| 153 |
+
return self._read_url_from_tunnel_stream()
|
| 154 |
+
|
| 155 |
+
def _read_url_from_tunnel_stream(self) -> str:
|
| 156 |
+
start_timestamp = time.time()
|
| 157 |
+
|
| 158 |
+
log = []
|
| 159 |
+
url = ""
|
| 160 |
+
|
| 161 |
+
def _raise_tunnel_error():
|
| 162 |
+
log_text = "\n".join(log)
|
| 163 |
+
print(log_text, file=sys.stderr)
|
| 164 |
+
raise ValueError(f"{TUNNEL_ERROR_MESSAGE}\n{log_text}")
|
| 165 |
+
|
| 166 |
+
while url == "":
|
| 167 |
+
if time.time() - start_timestamp >= TUNNEL_TIMEOUT_SECONDS:
|
| 168 |
+
_raise_tunnel_error()
|
| 169 |
+
|
| 170 |
+
assert self.proc is not None # noqa: S101
|
| 171 |
+
if self.proc.stdout is None:
|
| 172 |
+
continue
|
| 173 |
+
|
| 174 |
+
line = self.proc.stdout.readline()
|
| 175 |
+
line = line.decode("utf-8")
|
| 176 |
+
|
| 177 |
+
if line == "":
|
| 178 |
+
continue
|
| 179 |
+
|
| 180 |
+
log.append(line.strip())
|
| 181 |
+
|
| 182 |
+
if "start proxy success" in line:
|
| 183 |
+
result = re.search("start proxy success: (.+)\n", line)
|
| 184 |
+
if result is None:
|
| 185 |
+
_raise_tunnel_error()
|
| 186 |
+
else:
|
| 187 |
+
url = result.group(1)
|
| 188 |
+
elif "login to server failed" in line:
|
| 189 |
+
_raise_tunnel_error()
|
| 190 |
+
|
| 191 |
+
return url
|
trackio/alerts.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import logging
|
| 3 |
+
import ssl
|
| 4 |
+
import urllib.request
|
| 5 |
+
from enum import Enum
|
| 6 |
+
|
| 7 |
+
try:
|
| 8 |
+
import certifi
|
| 9 |
+
|
| 10 |
+
_SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
|
| 11 |
+
except ImportError:
|
| 12 |
+
_SSL_CONTEXT = None
|
| 13 |
+
|
| 14 |
+
logger = logging.getLogger(__name__)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class AlertLevel(str, Enum):
|
| 18 |
+
INFO = "info"
|
| 19 |
+
WARN = "warn"
|
| 20 |
+
ERROR = "error"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
ALERT_LEVEL_ORDER = {
|
| 24 |
+
AlertLevel.INFO: 0,
|
| 25 |
+
AlertLevel.WARN: 1,
|
| 26 |
+
AlertLevel.ERROR: 2,
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
ALERT_COLORS = {
|
| 30 |
+
AlertLevel.INFO: "\033[94m",
|
| 31 |
+
AlertLevel.WARN: "\033[93m",
|
| 32 |
+
AlertLevel.ERROR: "\033[91m",
|
| 33 |
+
}
|
| 34 |
+
RESET_COLOR = "\033[0m"
|
| 35 |
+
|
| 36 |
+
LEVEL_EMOJI = {
|
| 37 |
+
AlertLevel.INFO: "ℹ️",
|
| 38 |
+
AlertLevel.WARN: "⚠️",
|
| 39 |
+
AlertLevel.ERROR: "🚨",
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def format_alert_terminal(
|
| 44 |
+
level: AlertLevel, title: str, text: str | None, step: int | None
|
| 45 |
+
) -> str:
|
| 46 |
+
color = ALERT_COLORS.get(level, "")
|
| 47 |
+
step_str = f" (step {step})" if step is not None else ""
|
| 48 |
+
if text:
|
| 49 |
+
return f"{color}[TRACKIO {level.value.upper()}]{RESET_COLOR} {title}: {text}{step_str}"
|
| 50 |
+
return f"{color}[TRACKIO {level.value.upper()}]{RESET_COLOR} {title}{step_str}"
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _is_slack_url(url: str) -> bool:
|
| 54 |
+
return "hooks.slack.com" in url
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _is_discord_url(url: str) -> bool:
|
| 58 |
+
return "discord.com/api/webhooks" in url or "discordapp.com/api/webhooks" in url
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _build_slack_payload(
|
| 62 |
+
level: AlertLevel,
|
| 63 |
+
title: str,
|
| 64 |
+
text: str | None,
|
| 65 |
+
project: str,
|
| 66 |
+
run: str,
|
| 67 |
+
step: int | None,
|
| 68 |
+
) -> dict:
|
| 69 |
+
emoji = LEVEL_EMOJI.get(level, "")
|
| 70 |
+
step_str = f" • Step {step}" if step is not None else ""
|
| 71 |
+
header = f"{emoji} *[{level.value.upper()}] {title}*"
|
| 72 |
+
context = f"Project: {project} • Run: {run}{step_str}"
|
| 73 |
+
blocks = [
|
| 74 |
+
{"type": "section", "text": {"type": "mrkdwn", "text": header}},
|
| 75 |
+
]
|
| 76 |
+
if text:
|
| 77 |
+
blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": text}})
|
| 78 |
+
blocks.append(
|
| 79 |
+
{"type": "context", "elements": [{"type": "mrkdwn", "text": context}]}
|
| 80 |
+
)
|
| 81 |
+
return {"blocks": blocks}
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _build_discord_payload(
|
| 85 |
+
level: AlertLevel,
|
| 86 |
+
title: str,
|
| 87 |
+
text: str | None,
|
| 88 |
+
project: str,
|
| 89 |
+
run: str,
|
| 90 |
+
step: int | None,
|
| 91 |
+
) -> dict:
|
| 92 |
+
color_map = {
|
| 93 |
+
AlertLevel.INFO: 3447003,
|
| 94 |
+
AlertLevel.WARN: 16776960,
|
| 95 |
+
AlertLevel.ERROR: 15158332,
|
| 96 |
+
}
|
| 97 |
+
emoji = LEVEL_EMOJI.get(level, "")
|
| 98 |
+
step_str = f" • Step {step}" if step is not None else ""
|
| 99 |
+
embed = {
|
| 100 |
+
"title": f"{emoji} [{level.value.upper()}] {title}",
|
| 101 |
+
"color": color_map.get(level, 0),
|
| 102 |
+
"footer": {"text": f"Project: {project} • Run: {run}{step_str}"},
|
| 103 |
+
}
|
| 104 |
+
if text:
|
| 105 |
+
embed["description"] = text
|
| 106 |
+
return {"embeds": [embed]}
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _build_generic_payload(
|
| 110 |
+
level: AlertLevel,
|
| 111 |
+
title: str,
|
| 112 |
+
text: str | None,
|
| 113 |
+
project: str,
|
| 114 |
+
run: str,
|
| 115 |
+
step: int | None,
|
| 116 |
+
timestamp: str | None,
|
| 117 |
+
) -> dict:
|
| 118 |
+
return {
|
| 119 |
+
"level": level.value,
|
| 120 |
+
"title": title,
|
| 121 |
+
"text": text,
|
| 122 |
+
"project": project,
|
| 123 |
+
"run": run,
|
| 124 |
+
"step": step,
|
| 125 |
+
"timestamp": timestamp,
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def parse_alert_level(level: AlertLevel | str) -> AlertLevel:
|
| 130 |
+
if isinstance(level, AlertLevel):
|
| 131 |
+
return level
|
| 132 |
+
normalized = level.lower().strip()
|
| 133 |
+
try:
|
| 134 |
+
return AlertLevel(normalized)
|
| 135 |
+
except ValueError as e:
|
| 136 |
+
allowed = ", ".join(lvl.value for lvl in AlertLevel)
|
| 137 |
+
raise ValueError(
|
| 138 |
+
f"Invalid alert level '{level}'. Expected one of: {allowed}."
|
| 139 |
+
) from e
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def resolve_webhook_min_level(
|
| 143 |
+
webhook_min_level: AlertLevel | str | None,
|
| 144 |
+
) -> AlertLevel | None:
|
| 145 |
+
if webhook_min_level is None:
|
| 146 |
+
return None
|
| 147 |
+
return parse_alert_level(webhook_min_level)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def should_send_webhook(
|
| 151 |
+
level: AlertLevel, webhook_min_level: AlertLevel | None
|
| 152 |
+
) -> bool:
|
| 153 |
+
if webhook_min_level is None:
|
| 154 |
+
return True
|
| 155 |
+
return ALERT_LEVEL_ORDER[level] >= ALERT_LEVEL_ORDER[webhook_min_level]
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def send_webhook(
|
| 159 |
+
url: str,
|
| 160 |
+
level: AlertLevel,
|
| 161 |
+
title: str,
|
| 162 |
+
text: str | None,
|
| 163 |
+
project: str,
|
| 164 |
+
run: str,
|
| 165 |
+
step: int | None,
|
| 166 |
+
timestamp: str | None = None,
|
| 167 |
+
) -> None:
|
| 168 |
+
if _is_slack_url(url):
|
| 169 |
+
payload = _build_slack_payload(level, title, text, project, run, step)
|
| 170 |
+
elif _is_discord_url(url):
|
| 171 |
+
payload = _build_discord_payload(level, title, text, project, run, step)
|
| 172 |
+
else:
|
| 173 |
+
payload = _build_generic_payload(
|
| 174 |
+
level, title, text, project, run, step, timestamp
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
data = json.dumps(payload).encode("utf-8")
|
| 178 |
+
req = urllib.request.Request(
|
| 179 |
+
url, data=data, headers={"Content-Type": "application/json"}
|
| 180 |
+
)
|
| 181 |
+
try:
|
| 182 |
+
urllib.request.urlopen(req, timeout=10, context=_SSL_CONTEXT)
|
| 183 |
+
except Exception as e:
|
| 184 |
+
logger.warning(f"Failed to send webhook to {url}: {e}")
|
trackio/api.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Iterator
|
| 2 |
+
|
| 3 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class Run:
|
| 7 |
+
def __init__(self, project: str, name: str):
|
| 8 |
+
self.project = project
|
| 9 |
+
self.name = name
|
| 10 |
+
self._config = None
|
| 11 |
+
|
| 12 |
+
@property
|
| 13 |
+
def id(self) -> str:
|
| 14 |
+
return self.name
|
| 15 |
+
|
| 16 |
+
@property
|
| 17 |
+
def config(self) -> dict | None:
|
| 18 |
+
if self._config is None:
|
| 19 |
+
self._config = SQLiteStorage.get_run_config(self.project, self.name)
|
| 20 |
+
return self._config
|
| 21 |
+
|
| 22 |
+
def alerts(self, level: str | None = None, since: str | None = None) -> list[dict]:
|
| 23 |
+
return SQLiteStorage.get_alerts(
|
| 24 |
+
self.project, run_name=self.name, level=level, since=since
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
def delete(self) -> bool:
|
| 28 |
+
return SQLiteStorage.delete_run(self.project, self.name)
|
| 29 |
+
|
| 30 |
+
def move(self, new_project: str) -> bool:
|
| 31 |
+
success = SQLiteStorage.move_run(self.project, self.name, new_project)
|
| 32 |
+
if success:
|
| 33 |
+
self.project = new_project
|
| 34 |
+
return success
|
| 35 |
+
|
| 36 |
+
def rename(self, new_name: str) -> "Run":
|
| 37 |
+
SQLiteStorage.rename_run(self.project, self.name, new_name)
|
| 38 |
+
self.name = new_name
|
| 39 |
+
return self
|
| 40 |
+
|
| 41 |
+
def __repr__(self) -> str:
|
| 42 |
+
return f"<Run {self.name} in project {self.project}>"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class Runs:
|
| 46 |
+
def __init__(self, project: str):
|
| 47 |
+
self.project = project
|
| 48 |
+
self._runs = None
|
| 49 |
+
|
| 50 |
+
def _load_runs(self):
|
| 51 |
+
if self._runs is None:
|
| 52 |
+
run_names = SQLiteStorage.get_runs(self.project)
|
| 53 |
+
self._runs = [Run(self.project, name) for name in run_names]
|
| 54 |
+
|
| 55 |
+
def __iter__(self) -> Iterator[Run]:
|
| 56 |
+
self._load_runs()
|
| 57 |
+
return iter(self._runs)
|
| 58 |
+
|
| 59 |
+
def __getitem__(self, index: int) -> Run:
|
| 60 |
+
self._load_runs()
|
| 61 |
+
return self._runs[index]
|
| 62 |
+
|
| 63 |
+
def __len__(self) -> int:
|
| 64 |
+
self._load_runs()
|
| 65 |
+
return len(self._runs)
|
| 66 |
+
|
| 67 |
+
def __repr__(self) -> str:
|
| 68 |
+
self._load_runs()
|
| 69 |
+
return f"<Runs project={self.project} count={len(self._runs)}>"
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
class Api:
|
| 73 |
+
def runs(self, project: str) -> Runs:
|
| 74 |
+
if not SQLiteStorage.get_project_db_path(project).exists():
|
| 75 |
+
raise ValueError(f"Project '{project}' does not exist")
|
| 76 |
+
return Runs(project)
|
| 77 |
+
|
| 78 |
+
def alerts(
|
| 79 |
+
self,
|
| 80 |
+
project: str,
|
| 81 |
+
run: str | None = None,
|
| 82 |
+
level: str | None = None,
|
| 83 |
+
since: str | None = None,
|
| 84 |
+
) -> list[dict]:
|
| 85 |
+
if not SQLiteStorage.get_project_db_path(project).exists():
|
| 86 |
+
raise ValueError(f"Project '{project}' does not exist")
|
| 87 |
+
return SQLiteStorage.get_alerts(project, run_name=run, level=level, since=since)
|
trackio/apple_gpu.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import platform
|
| 2 |
+
import subprocess
|
| 3 |
+
import sys
|
| 4 |
+
import threading
|
| 5 |
+
import warnings
|
| 6 |
+
from typing import TYPE_CHECKING, Any
|
| 7 |
+
|
| 8 |
+
if TYPE_CHECKING:
|
| 9 |
+
from trackio.run import Run
|
| 10 |
+
|
| 11 |
+
psutil: Any = None
|
| 12 |
+
PSUTIL_AVAILABLE = False
|
| 13 |
+
_monitor_lock = threading.Lock()
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _ensure_psutil():
|
| 17 |
+
global PSUTIL_AVAILABLE, psutil
|
| 18 |
+
if PSUTIL_AVAILABLE:
|
| 19 |
+
return psutil
|
| 20 |
+
try:
|
| 21 |
+
import psutil as _psutil
|
| 22 |
+
|
| 23 |
+
psutil = _psutil
|
| 24 |
+
PSUTIL_AVAILABLE = True
|
| 25 |
+
return psutil
|
| 26 |
+
except ImportError:
|
| 27 |
+
raise ImportError(
|
| 28 |
+
"psutil is required for Apple Silicon monitoring. "
|
| 29 |
+
"Install it with: pip install psutil"
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def is_apple_silicon() -> bool:
|
| 34 |
+
"""Check if running on Apple Silicon (M1/M2/M3/M4)."""
|
| 35 |
+
if platform.system() != "Darwin":
|
| 36 |
+
return False
|
| 37 |
+
|
| 38 |
+
try:
|
| 39 |
+
result = subprocess.run(
|
| 40 |
+
["sysctl", "-n", "machdep.cpu.brand_string"],
|
| 41 |
+
capture_output=True,
|
| 42 |
+
text=True,
|
| 43 |
+
timeout=1,
|
| 44 |
+
)
|
| 45 |
+
cpu_brand = result.stdout.strip()
|
| 46 |
+
return "Apple" in cpu_brand
|
| 47 |
+
except Exception:
|
| 48 |
+
return False
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def get_gpu_info() -> dict[str, Any]:
|
| 52 |
+
"""Get Apple GPU information using ioreg."""
|
| 53 |
+
try:
|
| 54 |
+
result = subprocess.run(
|
| 55 |
+
["ioreg", "-r", "-d", "1", "-w", "0", "-c", "IOAccelerator"],
|
| 56 |
+
capture_output=True,
|
| 57 |
+
text=True,
|
| 58 |
+
timeout=2,
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
if result.returncode == 0 and result.stdout:
|
| 62 |
+
lines = result.stdout.strip().split("\n")
|
| 63 |
+
for line in lines:
|
| 64 |
+
if "IOAccelerator" in line and "class" in line:
|
| 65 |
+
return {"detected": True, "type": "Apple GPU"}
|
| 66 |
+
else:
|
| 67 |
+
print("Error collecting Apple GPU info. ioreg stdout was:", file=sys.stderr)
|
| 68 |
+
print(result.stdout, file=sys.stderr)
|
| 69 |
+
print("ioreg stderr was:", file=sys.stderr)
|
| 70 |
+
print(result.stderr, file=sys.stderr)
|
| 71 |
+
|
| 72 |
+
result = subprocess.run(
|
| 73 |
+
["system_profiler", "SPDisplaysDataType"],
|
| 74 |
+
capture_output=True,
|
| 75 |
+
text=True,
|
| 76 |
+
timeout=3,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
if result.returncode == 0 and "Apple" in result.stdout:
|
| 80 |
+
for line in result.stdout.split("\n"):
|
| 81 |
+
if "Chipset Model:" in line:
|
| 82 |
+
model = line.split(":")[-1].strip()
|
| 83 |
+
return {"detected": True, "type": model}
|
| 84 |
+
|
| 85 |
+
except Exception:
|
| 86 |
+
pass
|
| 87 |
+
|
| 88 |
+
return {"detected": False}
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def apple_gpu_available() -> bool:
|
| 92 |
+
"""
|
| 93 |
+
Check if Apple GPU monitoring is available.
|
| 94 |
+
|
| 95 |
+
Returns True if running on Apple Silicon (M-series chips) and psutil is installed.
|
| 96 |
+
"""
|
| 97 |
+
try:
|
| 98 |
+
_ensure_psutil()
|
| 99 |
+
return is_apple_silicon()
|
| 100 |
+
except ImportError:
|
| 101 |
+
return False
|
| 102 |
+
except Exception:
|
| 103 |
+
return False
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def collect_apple_metrics() -> dict:
|
| 107 |
+
"""
|
| 108 |
+
Collect system metrics for Apple Silicon.
|
| 109 |
+
|
| 110 |
+
Returns:
|
| 111 |
+
Dictionary of system metrics including CPU, memory, and GPU info.
|
| 112 |
+
"""
|
| 113 |
+
if not PSUTIL_AVAILABLE:
|
| 114 |
+
try:
|
| 115 |
+
_ensure_psutil()
|
| 116 |
+
except ImportError:
|
| 117 |
+
return {}
|
| 118 |
+
|
| 119 |
+
metrics = {}
|
| 120 |
+
|
| 121 |
+
try:
|
| 122 |
+
cpu_percent = psutil.cpu_percent(interval=0.1, percpu=False)
|
| 123 |
+
metrics["cpu/utilization"] = cpu_percent
|
| 124 |
+
except Exception:
|
| 125 |
+
pass
|
| 126 |
+
|
| 127 |
+
try:
|
| 128 |
+
cpu_percents = psutil.cpu_percent(interval=0.1, percpu=True)
|
| 129 |
+
for i, percent in enumerate(cpu_percents):
|
| 130 |
+
metrics[f"cpu/{i}/utilization"] = percent
|
| 131 |
+
except Exception:
|
| 132 |
+
pass
|
| 133 |
+
|
| 134 |
+
try:
|
| 135 |
+
cpu_freq = psutil.cpu_freq()
|
| 136 |
+
if cpu_freq:
|
| 137 |
+
metrics["cpu/frequency"] = cpu_freq.current
|
| 138 |
+
if cpu_freq.max > 0:
|
| 139 |
+
metrics["cpu/frequency_max"] = cpu_freq.max
|
| 140 |
+
except Exception:
|
| 141 |
+
pass
|
| 142 |
+
|
| 143 |
+
try:
|
| 144 |
+
mem = psutil.virtual_memory()
|
| 145 |
+
metrics["memory/used"] = mem.used / (1024**3)
|
| 146 |
+
metrics["memory/total"] = mem.total / (1024**3)
|
| 147 |
+
metrics["memory/available"] = mem.available / (1024**3)
|
| 148 |
+
metrics["memory/percent"] = mem.percent
|
| 149 |
+
except Exception:
|
| 150 |
+
pass
|
| 151 |
+
|
| 152 |
+
try:
|
| 153 |
+
swap = psutil.swap_memory()
|
| 154 |
+
metrics["swap/used"] = swap.used / (1024**3)
|
| 155 |
+
metrics["swap/total"] = swap.total / (1024**3)
|
| 156 |
+
metrics["swap/percent"] = swap.percent
|
| 157 |
+
except Exception:
|
| 158 |
+
pass
|
| 159 |
+
|
| 160 |
+
try:
|
| 161 |
+
sensors_temps = psutil.sensors_temperatures()
|
| 162 |
+
if sensors_temps:
|
| 163 |
+
for name, entries in sensors_temps.items():
|
| 164 |
+
for i, entry in enumerate(entries):
|
| 165 |
+
label = entry.label or f"{name}_{i}"
|
| 166 |
+
metrics[f"temp/{label}"] = entry.current
|
| 167 |
+
except Exception:
|
| 168 |
+
pass
|
| 169 |
+
|
| 170 |
+
gpu_info = get_gpu_info()
|
| 171 |
+
if gpu_info.get("detected"):
|
| 172 |
+
metrics["gpu/detected"] = 1
|
| 173 |
+
if "type" in gpu_info:
|
| 174 |
+
pass
|
| 175 |
+
|
| 176 |
+
return metrics
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class AppleGpuMonitor:
|
| 180 |
+
def __init__(self, run: "Run", interval: float = 10.0):
|
| 181 |
+
self._run = run
|
| 182 |
+
self._interval = interval
|
| 183 |
+
self._stop_flag = threading.Event()
|
| 184 |
+
self._thread: "threading.Thread | None" = None
|
| 185 |
+
|
| 186 |
+
def start(self):
|
| 187 |
+
if not is_apple_silicon():
|
| 188 |
+
warnings.warn(
|
| 189 |
+
"auto_log_gpu=True but not running on Apple Silicon. "
|
| 190 |
+
"Apple GPU logging disabled."
|
| 191 |
+
)
|
| 192 |
+
return
|
| 193 |
+
|
| 194 |
+
if not PSUTIL_AVAILABLE:
|
| 195 |
+
try:
|
| 196 |
+
_ensure_psutil()
|
| 197 |
+
except ImportError:
|
| 198 |
+
warnings.warn(
|
| 199 |
+
"auto_log_gpu=True but psutil not installed. "
|
| 200 |
+
"Install with: pip install psutil"
|
| 201 |
+
)
|
| 202 |
+
return
|
| 203 |
+
|
| 204 |
+
self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
|
| 205 |
+
self._thread.start()
|
| 206 |
+
|
| 207 |
+
def stop(self):
|
| 208 |
+
self._stop_flag.set()
|
| 209 |
+
if self._thread is not None:
|
| 210 |
+
self._thread.join(timeout=2.0)
|
| 211 |
+
|
| 212 |
+
def _monitor_loop(self):
|
| 213 |
+
while not self._stop_flag.is_set():
|
| 214 |
+
try:
|
| 215 |
+
metrics = collect_apple_metrics()
|
| 216 |
+
if metrics:
|
| 217 |
+
self._run.log_system(metrics)
|
| 218 |
+
except Exception:
|
| 219 |
+
pass
|
| 220 |
+
|
| 221 |
+
self._stop_flag.wait(timeout=self._interval)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def log_apple_gpu(run: "Run | None" = None) -> dict:
|
| 225 |
+
"""
|
| 226 |
+
Log Apple Silicon system metrics to the current or specified run.
|
| 227 |
+
|
| 228 |
+
Args:
|
| 229 |
+
run: Optional Run instance. If None, uses current run from context.
|
| 230 |
+
|
| 231 |
+
Returns:
|
| 232 |
+
dict: The system metrics that were logged.
|
| 233 |
+
|
| 234 |
+
Example:
|
| 235 |
+
```python
|
| 236 |
+
import trackio
|
| 237 |
+
|
| 238 |
+
run = trackio.init(project="my-project")
|
| 239 |
+
trackio.log({"loss": 0.5})
|
| 240 |
+
trackio.log_apple_gpu()
|
| 241 |
+
```
|
| 242 |
+
"""
|
| 243 |
+
from trackio import context_vars
|
| 244 |
+
|
| 245 |
+
if run is None:
|
| 246 |
+
run = context_vars.current_run.get()
|
| 247 |
+
if run is None:
|
| 248 |
+
raise RuntimeError("Call trackio.init() before trackio.log_apple_gpu().")
|
| 249 |
+
|
| 250 |
+
metrics = collect_apple_metrics()
|
| 251 |
+
if metrics:
|
| 252 |
+
run.log_system(metrics)
|
| 253 |
+
return metrics
|
trackio/asgi_app.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import asyncio
|
| 4 |
+
import inspect
|
| 5 |
+
import json
|
| 6 |
+
import secrets
|
| 7 |
+
import traceback
|
| 8 |
+
from collections.abc import Callable
|
| 9 |
+
from typing import Any
|
| 10 |
+
|
| 11 |
+
from starlette.applications import Starlette
|
| 12 |
+
from starlette.requests import Request
|
| 13 |
+
from starlette.responses import FileResponse, JSONResponse, Response, StreamingResponse
|
| 14 |
+
from starlette.routing import Route
|
| 15 |
+
|
| 16 |
+
from trackio.exceptions import TrackioAPIError
|
| 17 |
+
|
| 18 |
+
API_PREFIX = "/gradio_api"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _serialize_result(data: Any) -> str:
|
| 22 |
+
def default(o: Any):
|
| 23 |
+
if isinstance(o, (dict, list, str, int, float, bool)) or o is None:
|
| 24 |
+
return o
|
| 25 |
+
if hasattr(o, "item"):
|
| 26 |
+
try:
|
| 27 |
+
return o.item()
|
| 28 |
+
except Exception:
|
| 29 |
+
pass
|
| 30 |
+
return str(o)
|
| 31 |
+
|
| 32 |
+
return json.dumps(data, default=default)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _invoke_handler(
|
| 36 |
+
fn: Callable,
|
| 37 |
+
request: Request,
|
| 38 |
+
data: list[Any],
|
| 39 |
+
) -> Any:
|
| 40 |
+
sig = inspect.signature(fn)
|
| 41 |
+
params = list(sig.parameters.values())
|
| 42 |
+
pos_args: list[Any] = []
|
| 43 |
+
di = 0
|
| 44 |
+
for p in params:
|
| 45 |
+
if p.name == "request":
|
| 46 |
+
pos_args.append(request)
|
| 47 |
+
else:
|
| 48 |
+
if di < len(data):
|
| 49 |
+
pos_args.append(data[di])
|
| 50 |
+
else:
|
| 51 |
+
pos_args.append(None)
|
| 52 |
+
di += 1
|
| 53 |
+
return fn(*pos_args)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class _EventStore:
|
| 57 |
+
def __init__(self) -> None:
|
| 58 |
+
self._results: dict[str, tuple[Any, bool, str | None]] = {}
|
| 59 |
+
|
| 60 |
+
def store_result(
|
| 61 |
+
self, event_id: str, result: Any, is_error: bool, err_msg: str | None
|
| 62 |
+
) -> None:
|
| 63 |
+
self._results[event_id] = (result, is_error, err_msg)
|
| 64 |
+
|
| 65 |
+
def pop(self, event_id: str) -> tuple[Any, bool, str | None] | None:
|
| 66 |
+
return self._results.pop(event_id, None)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
_event_store = _EventStore()
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def build_gradio_compat_handlers(
|
| 73 |
+
api_registry: dict[str, Callable[..., Any]],
|
| 74 |
+
) -> tuple[Callable, Callable]:
|
| 75 |
+
async def call_post(request: Request) -> Response:
|
| 76 |
+
api_name = request.path_params["api_name"].lstrip("/")
|
| 77 |
+
fn = api_registry.get(api_name)
|
| 78 |
+
if fn is None:
|
| 79 |
+
return JSONResponse({"detail": f"Unknown API: {api_name}"}, status_code=404)
|
| 80 |
+
try:
|
| 81 |
+
body = await request.json()
|
| 82 |
+
except Exception:
|
| 83 |
+
body = {}
|
| 84 |
+
data = body.get("data") or []
|
| 85 |
+
if not isinstance(data, list):
|
| 86 |
+
data = [data]
|
| 87 |
+
event_id = secrets.token_urlsafe(16)
|
| 88 |
+
try:
|
| 89 |
+
out = await asyncio.to_thread(_invoke_handler, fn, request, data)
|
| 90 |
+
_event_store.store_result(event_id, out, False, None)
|
| 91 |
+
except TrackioAPIError as e:
|
| 92 |
+
_event_store.store_result(event_id, None, True, str(e))
|
| 93 |
+
except Exception:
|
| 94 |
+
tb = traceback.format_exc()
|
| 95 |
+
print(tb)
|
| 96 |
+
_event_store.store_result(event_id, None, True, tb)
|
| 97 |
+
return JSONResponse({"event_id": event_id})
|
| 98 |
+
|
| 99 |
+
async def call_get(request: Request) -> Response:
|
| 100 |
+
event_id = request.path_params["event_id"]
|
| 101 |
+
|
| 102 |
+
async def sse_gen():
|
| 103 |
+
for _ in range(500):
|
| 104 |
+
await asyncio.sleep(0.005)
|
| 105 |
+
got = _event_store.pop(event_id)
|
| 106 |
+
if got is not None:
|
| 107 |
+
result, is_err, err_msg = got
|
| 108 |
+
if is_err:
|
| 109 |
+
payload = json.dumps(err_msg)
|
| 110 |
+
yield f"event: error\ndata: {payload}\n\n"
|
| 111 |
+
else:
|
| 112 |
+
wrapped = [result] if not isinstance(result, list) else result
|
| 113 |
+
payload = _serialize_result(wrapped)
|
| 114 |
+
yield f"event: complete\ndata: {payload}\n\n"
|
| 115 |
+
return
|
| 116 |
+
err = json.dumps("timeout waiting for result")
|
| 117 |
+
yield f"event: error\ndata: {err}\n\n"
|
| 118 |
+
|
| 119 |
+
return StreamingResponse(sse_gen(), media_type="text/event-stream")
|
| 120 |
+
|
| 121 |
+
return call_post, call_get
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
async def gradio_file_handler(request: Request) -> Response:
|
| 125 |
+
from pathlib import Path
|
| 126 |
+
from urllib.parse import unquote
|
| 127 |
+
|
| 128 |
+
rest = request.path_params.get("rest", "")
|
| 129 |
+
if not rest.startswith("file="):
|
| 130 |
+
return Response("Not found", status_code=404)
|
| 131 |
+
fs_path = unquote(rest[5:])
|
| 132 |
+
fp = Path(fs_path)
|
| 133 |
+
if fp.is_file():
|
| 134 |
+
return FileResponse(str(fp))
|
| 135 |
+
return Response("Not found", status_code=404)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
async def startup_events_handler(request: Request) -> Response:
|
| 139 |
+
return JSONResponse({"status": "ok"})
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def create_trackio_starlette_app(
|
| 143 |
+
oauth_routes: list[Route],
|
| 144 |
+
api_registry: dict[str, Callable[..., Any]],
|
| 145 |
+
mcp_lifespan: Any = None,
|
| 146 |
+
) -> Starlette:
|
| 147 |
+
call_post, call_get = build_gradio_compat_handlers(api_registry)
|
| 148 |
+
routes: list[Any] = list(oauth_routes)
|
| 149 |
+
routes.append(
|
| 150 |
+
Route(
|
| 151 |
+
API_PREFIX + "/call/{api_name:path}",
|
| 152 |
+
endpoint=call_post,
|
| 153 |
+
methods=["POST"],
|
| 154 |
+
)
|
| 155 |
+
)
|
| 156 |
+
routes.append(
|
| 157 |
+
Route(
|
| 158 |
+
API_PREFIX + "/call/{api_name:path}/{event_id}",
|
| 159 |
+
endpoint=call_get,
|
| 160 |
+
methods=["GET"],
|
| 161 |
+
)
|
| 162 |
+
)
|
| 163 |
+
routes.append(
|
| 164 |
+
Route(
|
| 165 |
+
API_PREFIX + "/startup-events",
|
| 166 |
+
endpoint=startup_events_handler,
|
| 167 |
+
methods=["GET"],
|
| 168 |
+
)
|
| 169 |
+
)
|
| 170 |
+
routes.append(
|
| 171 |
+
Route(
|
| 172 |
+
API_PREFIX + "/{rest:path}", endpoint=gradio_file_handler, methods=["GET"]
|
| 173 |
+
)
|
| 174 |
+
)
|
| 175 |
+
return Starlette(routes=routes, lifespan=mcp_lifespan)
|
trackio/assets/badge.png
ADDED
|
Git LFS Details
|
trackio/assets/trackio_logo_dark.png
ADDED
|
trackio/assets/trackio_logo_light.png
ADDED
|
trackio/assets/trackio_logo_old.png
ADDED
|
Git LFS Details
|
trackio/assets/trackio_logo_type_dark.png
ADDED
|
trackio/assets/trackio_logo_type_dark_transparent.png
ADDED
|
trackio/assets/trackio_logo_type_light.png
ADDED
|
trackio/assets/trackio_logo_type_light_transparent.png
ADDED
|
trackio/bucket_storage.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import shutil
|
| 2 |
+
import tempfile
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
import huggingface_hub
|
| 6 |
+
from huggingface_hub import copy_files, sync_bucket
|
| 7 |
+
|
| 8 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 9 |
+
from trackio.utils import MEDIA_DIR, TRACKIO_DIR
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def create_bucket_if_not_exists(bucket_id: str, private: bool | None = None) -> None:
|
| 13 |
+
huggingface_hub.create_bucket(bucket_id, private=private, exist_ok=True)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _list_bucket_file_paths(bucket_id: str, prefix: str | None = None) -> list[str]:
|
| 17 |
+
items = huggingface_hub.list_bucket_tree(bucket_id, prefix=prefix, recursive=True)
|
| 18 |
+
return [
|
| 19 |
+
item.path
|
| 20 |
+
for item in items
|
| 21 |
+
if getattr(item, "type", None) == "file" and getattr(item, "path", None)
|
| 22 |
+
]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def download_bucket_to_trackio_dir(bucket_id: str) -> None:
|
| 26 |
+
TRACKIO_DIR.mkdir(parents=True, exist_ok=True)
|
| 27 |
+
sync_bucket(
|
| 28 |
+
source=f"hf://buckets/{bucket_id}",
|
| 29 |
+
dest=str(TRACKIO_DIR.parent),
|
| 30 |
+
quiet=True,
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def upload_project_to_bucket(project: str, bucket_id: str) -> None:
|
| 35 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 36 |
+
if not db_path.exists():
|
| 37 |
+
raise FileNotFoundError(f"No database found for project '{project}'")
|
| 38 |
+
|
| 39 |
+
with SQLiteStorage._get_connection(
|
| 40 |
+
db_path, configure_pragmas=False, row_factory=None
|
| 41 |
+
) as conn:
|
| 42 |
+
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
| 43 |
+
|
| 44 |
+
files_to_add = [(str(db_path), f"trackio/{db_path.name}")]
|
| 45 |
+
|
| 46 |
+
media_dir = MEDIA_DIR / project
|
| 47 |
+
if media_dir.exists():
|
| 48 |
+
for media_file in media_dir.rglob("*"):
|
| 49 |
+
if media_file.is_file():
|
| 50 |
+
rel = media_file.relative_to(TRACKIO_DIR)
|
| 51 |
+
files_to_add.append((str(media_file), f"trackio/{rel}"))
|
| 52 |
+
|
| 53 |
+
huggingface_hub.batch_bucket_files(bucket_id, add=files_to_add)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _download_db_from_bucket(
|
| 57 |
+
project: str, bucket_id: str, dest_path: Path | None = None
|
| 58 |
+
) -> bool:
|
| 59 |
+
db_filename = SQLiteStorage.get_project_db_filename(project)
|
| 60 |
+
remote_path = f"trackio/{db_filename}"
|
| 61 |
+
local_path = dest_path or SQLiteStorage.get_project_db_path(project)
|
| 62 |
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
| 63 |
+
try:
|
| 64 |
+
huggingface_hub.download_bucket_files(
|
| 65 |
+
bucket_id,
|
| 66 |
+
files=[(remote_path, str(local_path))],
|
| 67 |
+
)
|
| 68 |
+
return local_path.exists()
|
| 69 |
+
except Exception:
|
| 70 |
+
return False
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _local_db_has_data(project: str) -> bool:
|
| 74 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 75 |
+
if not db_path.exists() or db_path.stat().st_size == 0:
|
| 76 |
+
return False
|
| 77 |
+
try:
|
| 78 |
+
with SQLiteStorage._get_connection(
|
| 79 |
+
db_path, configure_pragmas=False, row_factory=None
|
| 80 |
+
) as conn:
|
| 81 |
+
count = conn.execute("SELECT COUNT(*) FROM metrics").fetchone()[0]
|
| 82 |
+
return count > 0
|
| 83 |
+
except Exception:
|
| 84 |
+
return False
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _export_and_upload_static(
|
| 88 |
+
project: str,
|
| 89 |
+
dest_bucket_id: str,
|
| 90 |
+
db_path: Path,
|
| 91 |
+
media_dir: Path | None = None,
|
| 92 |
+
) -> None:
|
| 93 |
+
with tempfile.TemporaryDirectory() as tmp_dir:
|
| 94 |
+
output_dir = Path(tmp_dir)
|
| 95 |
+
SQLiteStorage.export_for_static_space(
|
| 96 |
+
project, output_dir, db_path_override=db_path
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
if media_dir and media_dir.exists():
|
| 100 |
+
shutil.copytree(media_dir, output_dir / "media")
|
| 101 |
+
|
| 102 |
+
files_to_add = []
|
| 103 |
+
for f in output_dir.rglob("*"):
|
| 104 |
+
if f.is_file():
|
| 105 |
+
rel = f.relative_to(output_dir)
|
| 106 |
+
files_to_add.append((str(f), str(rel)))
|
| 107 |
+
|
| 108 |
+
huggingface_hub.batch_bucket_files(dest_bucket_id, add=files_to_add)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _copy_project_media_between_buckets(
|
| 112 |
+
source_bucket_id: str, dest_bucket_id: str, project: str
|
| 113 |
+
) -> None:
|
| 114 |
+
source_media_prefix = f"trackio/media/{project}/"
|
| 115 |
+
media_to_copy = _list_bucket_file_paths(
|
| 116 |
+
source_bucket_id, prefix=source_media_prefix
|
| 117 |
+
)
|
| 118 |
+
if not media_to_copy:
|
| 119 |
+
return
|
| 120 |
+
|
| 121 |
+
copy_files(
|
| 122 |
+
f"hf://buckets/{source_bucket_id}/{source_media_prefix}",
|
| 123 |
+
f"hf://buckets/{dest_bucket_id}/media/",
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def upload_project_to_bucket_for_static(project: str, bucket_id: str) -> None:
|
| 128 |
+
if not _local_db_has_data(project):
|
| 129 |
+
_download_db_from_bucket(project, bucket_id)
|
| 130 |
+
|
| 131 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 132 |
+
_export_and_upload_static(project, bucket_id, db_path, MEDIA_DIR / project)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def export_from_bucket_for_static(
|
| 136 |
+
source_bucket_id: str,
|
| 137 |
+
dest_bucket_id: str,
|
| 138 |
+
project: str,
|
| 139 |
+
) -> None:
|
| 140 |
+
with tempfile.TemporaryDirectory() as work_dir:
|
| 141 |
+
work_path = Path(work_dir)
|
| 142 |
+
db_path = work_path / SQLiteStorage.get_project_db_filename(project)
|
| 143 |
+
|
| 144 |
+
if not _download_db_from_bucket(project, source_bucket_id, dest_path=db_path):
|
| 145 |
+
raise FileNotFoundError(
|
| 146 |
+
f"Could not download database for project '{project}' "
|
| 147 |
+
f"from bucket '{source_bucket_id}'."
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
_export_and_upload_static(project, dest_bucket_id, db_path)
|
| 151 |
+
_copy_project_media_between_buckets(source_bucket_id, dest_bucket_id, project)
|
trackio/cli.py
ADDED
|
@@ -0,0 +1,1272 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import os
|
| 3 |
+
|
| 4 |
+
from trackio import freeze, show, sync
|
| 5 |
+
from trackio.cli_helpers import (
|
| 6 |
+
error_exit,
|
| 7 |
+
format_alerts,
|
| 8 |
+
format_json,
|
| 9 |
+
format_list,
|
| 10 |
+
format_metric_values,
|
| 11 |
+
format_project_summary,
|
| 12 |
+
format_run_summary,
|
| 13 |
+
format_snapshot,
|
| 14 |
+
format_system_metric_names,
|
| 15 |
+
format_system_metrics,
|
| 16 |
+
)
|
| 17 |
+
from trackio.markdown import Markdown
|
| 18 |
+
from trackio.server import get_project_summary, get_run_summary
|
| 19 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _get_space(args):
|
| 23 |
+
return getattr(args, "space", None)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _get_remote(args):
|
| 27 |
+
from trackio.remote_client import RemoteClient
|
| 28 |
+
|
| 29 |
+
space = _get_space(args)
|
| 30 |
+
if not space:
|
| 31 |
+
return None
|
| 32 |
+
hf_token = getattr(args, "hf_token", None)
|
| 33 |
+
return RemoteClient(space, hf_token=hf_token)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _handle_status():
|
| 37 |
+
print("Reading local Trackio projects...\n")
|
| 38 |
+
projects = SQLiteStorage.get_projects()
|
| 39 |
+
if not projects:
|
| 40 |
+
print("No Trackio projects found.")
|
| 41 |
+
return
|
| 42 |
+
|
| 43 |
+
local_projects = []
|
| 44 |
+
synced_projects = []
|
| 45 |
+
unsynced_projects = []
|
| 46 |
+
|
| 47 |
+
for project in projects:
|
| 48 |
+
space_id = SQLiteStorage.get_space_id(project)
|
| 49 |
+
if space_id is None:
|
| 50 |
+
local_projects.append(project)
|
| 51 |
+
elif SQLiteStorage.has_pending_data(project):
|
| 52 |
+
unsynced_projects.append(project)
|
| 53 |
+
else:
|
| 54 |
+
synced_projects.append(project)
|
| 55 |
+
|
| 56 |
+
print("Finished reading Trackio projects")
|
| 57 |
+
if local_projects:
|
| 58 |
+
print(f" * {len(local_projects)} local trackio project(s) [OK]")
|
| 59 |
+
if synced_projects:
|
| 60 |
+
print(f" * {len(synced_projects)} trackio project(s) synced to Spaces [OK]")
|
| 61 |
+
if unsynced_projects:
|
| 62 |
+
print(
|
| 63 |
+
f" * {len(unsynced_projects)} trackio project(s) with unsynced changes [WARNING]:"
|
| 64 |
+
)
|
| 65 |
+
for p in unsynced_projects:
|
| 66 |
+
print(f" - {p}")
|
| 67 |
+
|
| 68 |
+
if unsynced_projects:
|
| 69 |
+
print(
|
| 70 |
+
f"\nRun `trackio sync --project {unsynced_projects[0]}` to sync. "
|
| 71 |
+
"Or run `trackio sync --all` to sync all unsynced changes."
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _handle_sync(args):
|
| 76 |
+
from trackio.deploy import sync_incremental
|
| 77 |
+
|
| 78 |
+
if args.sync_all and args.project:
|
| 79 |
+
error_exit("Cannot use --all and --project together.")
|
| 80 |
+
if not args.sync_all and not args.project:
|
| 81 |
+
error_exit("Must provide either --project or --all.")
|
| 82 |
+
|
| 83 |
+
if args.sync_all:
|
| 84 |
+
projects = SQLiteStorage.get_projects()
|
| 85 |
+
synced_any = False
|
| 86 |
+
for project in projects:
|
| 87 |
+
space_id = SQLiteStorage.get_space_id(project)
|
| 88 |
+
if space_id and SQLiteStorage.has_pending_data(project):
|
| 89 |
+
sync_incremental(
|
| 90 |
+
project, space_id, private=args.private, pending_only=True
|
| 91 |
+
)
|
| 92 |
+
synced_any = True
|
| 93 |
+
if not synced_any:
|
| 94 |
+
print("No projects with unsynced data found.")
|
| 95 |
+
else:
|
| 96 |
+
space_id = args.space_id
|
| 97 |
+
if space_id is None:
|
| 98 |
+
space_id = SQLiteStorage.get_space_id(args.project)
|
| 99 |
+
sync(
|
| 100 |
+
project=args.project,
|
| 101 |
+
space_id=space_id,
|
| 102 |
+
private=args.private,
|
| 103 |
+
force=args.force,
|
| 104 |
+
sdk=args.sdk,
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _extract_reports(
|
| 109 |
+
run: str, logs: list[dict], report_name: str | None = None
|
| 110 |
+
) -> list[dict]:
|
| 111 |
+
reports = []
|
| 112 |
+
for log in logs:
|
| 113 |
+
timestamp = log.get("timestamp")
|
| 114 |
+
step = log.get("step")
|
| 115 |
+
for key, value in log.items():
|
| 116 |
+
if report_name is not None and key != report_name:
|
| 117 |
+
continue
|
| 118 |
+
if isinstance(value, dict) and value.get("_type") == Markdown.TYPE:
|
| 119 |
+
content = value.get("_value")
|
| 120 |
+
if isinstance(content, str):
|
| 121 |
+
reports.append(
|
| 122 |
+
{
|
| 123 |
+
"run": run,
|
| 124 |
+
"report": key,
|
| 125 |
+
"step": step,
|
| 126 |
+
"timestamp": timestamp,
|
| 127 |
+
"content": content,
|
| 128 |
+
}
|
| 129 |
+
)
|
| 130 |
+
return reports
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def main():
|
| 134 |
+
parser = argparse.ArgumentParser(description="Trackio CLI")
|
| 135 |
+
parser.add_argument(
|
| 136 |
+
"--space",
|
| 137 |
+
required=False,
|
| 138 |
+
help="HF Space ID (e.g. 'user/space') or Space URL to query remotely.",
|
| 139 |
+
)
|
| 140 |
+
parser.add_argument(
|
| 141 |
+
"--hf-token",
|
| 142 |
+
required=False,
|
| 143 |
+
help="HF token for accessing private Spaces.",
|
| 144 |
+
)
|
| 145 |
+
subparsers = parser.add_subparsers(dest="command")
|
| 146 |
+
|
| 147 |
+
ui_parser = subparsers.add_parser(
|
| 148 |
+
"show", help="Show the Trackio dashboard UI for a project"
|
| 149 |
+
)
|
| 150 |
+
ui_parser.add_argument(
|
| 151 |
+
"--project", required=False, help="Project name to show in the dashboard"
|
| 152 |
+
)
|
| 153 |
+
ui_parser.add_argument(
|
| 154 |
+
"--theme",
|
| 155 |
+
required=False,
|
| 156 |
+
default="default",
|
| 157 |
+
help="A Gradio Theme to use for the dashboard instead of the default, can be a built-in theme (e.g. 'soft', 'citrus'), or a theme from the Hub (e.g. 'gstaff/xkcd').",
|
| 158 |
+
)
|
| 159 |
+
ui_parser.add_argument(
|
| 160 |
+
"--mcp-server",
|
| 161 |
+
action="store_true",
|
| 162 |
+
help="Enable MCP server functionality. The Trackio dashboard will be set up as an MCP server and certain functions will be exposed as MCP tools.",
|
| 163 |
+
)
|
| 164 |
+
ui_parser.add_argument(
|
| 165 |
+
"--footer",
|
| 166 |
+
action="store_true",
|
| 167 |
+
default=True,
|
| 168 |
+
help="Show the Gradio footer. Use --no-footer to hide it.",
|
| 169 |
+
)
|
| 170 |
+
ui_parser.add_argument(
|
| 171 |
+
"--no-footer",
|
| 172 |
+
dest="footer",
|
| 173 |
+
action="store_false",
|
| 174 |
+
help="Hide the Gradio footer.",
|
| 175 |
+
)
|
| 176 |
+
ui_parser.add_argument(
|
| 177 |
+
"--color-palette",
|
| 178 |
+
required=False,
|
| 179 |
+
help="Comma-separated list of hex color codes for plot lines (e.g. '#FF0000,#00FF00,#0000FF'). If not provided, the TRACKIO_COLOR_PALETTE environment variable will be used, or the default palette if not set.",
|
| 180 |
+
)
|
| 181 |
+
ui_parser.add_argument(
|
| 182 |
+
"--host",
|
| 183 |
+
required=False,
|
| 184 |
+
help="Host to bind the server to (e.g. '0.0.0.0' for remote access). If not provided, defaults to '127.0.0.1' (localhost only).",
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
subparsers.add_parser(
|
| 188 |
+
"status",
|
| 189 |
+
help="Show the status of all local Trackio projects, including sync status.",
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
sync_parser = subparsers.add_parser(
|
| 193 |
+
"sync",
|
| 194 |
+
help="Sync a local project's database to a Hugging Face Space. If the Space does not exist, it will be created.",
|
| 195 |
+
)
|
| 196 |
+
sync_parser.add_argument(
|
| 197 |
+
"--project",
|
| 198 |
+
required=False,
|
| 199 |
+
help="The name of the local project.",
|
| 200 |
+
)
|
| 201 |
+
sync_parser.add_argument(
|
| 202 |
+
"--space-id",
|
| 203 |
+
required=False,
|
| 204 |
+
help="The Hugging Face Space ID where the project will be synced (e.g. username/space_id). If not provided, uses the previously-configured Space.",
|
| 205 |
+
)
|
| 206 |
+
sync_parser.add_argument(
|
| 207 |
+
"--all",
|
| 208 |
+
action="store_true",
|
| 209 |
+
dest="sync_all",
|
| 210 |
+
help="Sync all projects that have unsynced data to their configured Spaces.",
|
| 211 |
+
)
|
| 212 |
+
sync_parser.add_argument(
|
| 213 |
+
"--private",
|
| 214 |
+
action="store_true",
|
| 215 |
+
help="Make the Hugging Face Space private if creating a new Space. By default, the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.",
|
| 216 |
+
)
|
| 217 |
+
sync_parser.add_argument(
|
| 218 |
+
"--force",
|
| 219 |
+
action="store_true",
|
| 220 |
+
help="Overwrite the existing database without prompting for confirmation.",
|
| 221 |
+
)
|
| 222 |
+
sync_parser.add_argument(
|
| 223 |
+
"--sdk",
|
| 224 |
+
choices=["gradio", "static"],
|
| 225 |
+
default="gradio",
|
| 226 |
+
help="The type of Space to deploy. 'gradio' (default) deploys a live Gradio server. 'static' deploys a static Space that reads from an HF Bucket.",
|
| 227 |
+
)
|
| 228 |
+
|
| 229 |
+
freeze_parser = subparsers.add_parser(
|
| 230 |
+
"freeze",
|
| 231 |
+
help="Create a one-time static Space snapshot from a project's data.",
|
| 232 |
+
)
|
| 233 |
+
freeze_parser.add_argument(
|
| 234 |
+
"--space-id",
|
| 235 |
+
required=True,
|
| 236 |
+
help="The source Gradio Space ID (e.g. username/space_id).",
|
| 237 |
+
)
|
| 238 |
+
freeze_parser.add_argument(
|
| 239 |
+
"--project",
|
| 240 |
+
required=True,
|
| 241 |
+
help="The name of the project to freeze into a static snapshot.",
|
| 242 |
+
)
|
| 243 |
+
freeze_parser.add_argument(
|
| 244 |
+
"--new-space-id",
|
| 245 |
+
required=False,
|
| 246 |
+
help="The Space ID for the new static Space. Defaults to {space_id}_static.",
|
| 247 |
+
)
|
| 248 |
+
freeze_parser.add_argument(
|
| 249 |
+
"--private",
|
| 250 |
+
action="store_true",
|
| 251 |
+
help="Make the new static Space private.",
|
| 252 |
+
)
|
| 253 |
+
|
| 254 |
+
list_parser = subparsers.add_parser(
|
| 255 |
+
"list",
|
| 256 |
+
help="List projects, runs, or metrics",
|
| 257 |
+
)
|
| 258 |
+
list_subparsers = list_parser.add_subparsers(dest="list_type", required=True)
|
| 259 |
+
|
| 260 |
+
list_projects_parser = list_subparsers.add_parser(
|
| 261 |
+
"projects",
|
| 262 |
+
help="List all projects",
|
| 263 |
+
)
|
| 264 |
+
list_projects_parser.add_argument(
|
| 265 |
+
"--json",
|
| 266 |
+
action="store_true",
|
| 267 |
+
help="Output in JSON format",
|
| 268 |
+
)
|
| 269 |
+
|
| 270 |
+
list_runs_parser = list_subparsers.add_parser(
|
| 271 |
+
"runs",
|
| 272 |
+
help="List runs for a project",
|
| 273 |
+
)
|
| 274 |
+
list_runs_parser.add_argument(
|
| 275 |
+
"--project",
|
| 276 |
+
required=True,
|
| 277 |
+
help="Project name",
|
| 278 |
+
)
|
| 279 |
+
list_runs_parser.add_argument(
|
| 280 |
+
"--json",
|
| 281 |
+
action="store_true",
|
| 282 |
+
help="Output in JSON format",
|
| 283 |
+
)
|
| 284 |
+
|
| 285 |
+
list_metrics_parser = list_subparsers.add_parser(
|
| 286 |
+
"metrics",
|
| 287 |
+
help="List metrics for a run",
|
| 288 |
+
)
|
| 289 |
+
list_metrics_parser.add_argument(
|
| 290 |
+
"--project",
|
| 291 |
+
required=True,
|
| 292 |
+
help="Project name",
|
| 293 |
+
)
|
| 294 |
+
list_metrics_parser.add_argument(
|
| 295 |
+
"--run",
|
| 296 |
+
required=True,
|
| 297 |
+
help="Run name",
|
| 298 |
+
)
|
| 299 |
+
list_metrics_parser.add_argument(
|
| 300 |
+
"--json",
|
| 301 |
+
action="store_true",
|
| 302 |
+
help="Output in JSON format",
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
list_system_metrics_parser = list_subparsers.add_parser(
|
| 306 |
+
"system-metrics",
|
| 307 |
+
help="List system metrics for a run",
|
| 308 |
+
)
|
| 309 |
+
list_system_metrics_parser.add_argument(
|
| 310 |
+
"--project",
|
| 311 |
+
required=True,
|
| 312 |
+
help="Project name",
|
| 313 |
+
)
|
| 314 |
+
list_system_metrics_parser.add_argument(
|
| 315 |
+
"--run",
|
| 316 |
+
required=True,
|
| 317 |
+
help="Run name",
|
| 318 |
+
)
|
| 319 |
+
list_system_metrics_parser.add_argument(
|
| 320 |
+
"--json",
|
| 321 |
+
action="store_true",
|
| 322 |
+
help="Output in JSON format",
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
list_alerts_parser = list_subparsers.add_parser(
|
| 326 |
+
"alerts",
|
| 327 |
+
help="List alerts for a project or run",
|
| 328 |
+
)
|
| 329 |
+
list_alerts_parser.add_argument(
|
| 330 |
+
"--project",
|
| 331 |
+
required=True,
|
| 332 |
+
help="Project name",
|
| 333 |
+
)
|
| 334 |
+
list_alerts_parser.add_argument(
|
| 335 |
+
"--run",
|
| 336 |
+
required=False,
|
| 337 |
+
help="Run name (optional)",
|
| 338 |
+
)
|
| 339 |
+
list_alerts_parser.add_argument(
|
| 340 |
+
"--level",
|
| 341 |
+
required=False,
|
| 342 |
+
help="Filter by alert level (info, warn, error)",
|
| 343 |
+
)
|
| 344 |
+
list_alerts_parser.add_argument(
|
| 345 |
+
"--json",
|
| 346 |
+
action="store_true",
|
| 347 |
+
help="Output in JSON format",
|
| 348 |
+
)
|
| 349 |
+
list_alerts_parser.add_argument(
|
| 350 |
+
"--since",
|
| 351 |
+
required=False,
|
| 352 |
+
help="Only show alerts after this ISO 8601 timestamp",
|
| 353 |
+
)
|
| 354 |
+
|
| 355 |
+
list_reports_parser = list_subparsers.add_parser(
|
| 356 |
+
"reports",
|
| 357 |
+
help="List markdown reports for a project or run",
|
| 358 |
+
)
|
| 359 |
+
list_reports_parser.add_argument(
|
| 360 |
+
"--project",
|
| 361 |
+
required=True,
|
| 362 |
+
help="Project name",
|
| 363 |
+
)
|
| 364 |
+
list_reports_parser.add_argument(
|
| 365 |
+
"--run",
|
| 366 |
+
required=False,
|
| 367 |
+
help="Run name (optional)",
|
| 368 |
+
)
|
| 369 |
+
list_reports_parser.add_argument(
|
| 370 |
+
"--json",
|
| 371 |
+
action="store_true",
|
| 372 |
+
help="Output in JSON format",
|
| 373 |
+
)
|
| 374 |
+
|
| 375 |
+
get_parser = subparsers.add_parser(
|
| 376 |
+
"get",
|
| 377 |
+
help="Get project, run, or metric information",
|
| 378 |
+
)
|
| 379 |
+
get_subparsers = get_parser.add_subparsers(dest="get_type", required=True)
|
| 380 |
+
|
| 381 |
+
get_project_parser = get_subparsers.add_parser(
|
| 382 |
+
"project",
|
| 383 |
+
help="Get project summary",
|
| 384 |
+
)
|
| 385 |
+
get_project_parser.add_argument(
|
| 386 |
+
"--project",
|
| 387 |
+
required=True,
|
| 388 |
+
help="Project name",
|
| 389 |
+
)
|
| 390 |
+
get_project_parser.add_argument(
|
| 391 |
+
"--json",
|
| 392 |
+
action="store_true",
|
| 393 |
+
help="Output in JSON format",
|
| 394 |
+
)
|
| 395 |
+
|
| 396 |
+
get_run_parser = get_subparsers.add_parser(
|
| 397 |
+
"run",
|
| 398 |
+
help="Get run summary",
|
| 399 |
+
)
|
| 400 |
+
get_run_parser.add_argument(
|
| 401 |
+
"--project",
|
| 402 |
+
required=True,
|
| 403 |
+
help="Project name",
|
| 404 |
+
)
|
| 405 |
+
get_run_parser.add_argument(
|
| 406 |
+
"--run",
|
| 407 |
+
required=True,
|
| 408 |
+
help="Run name",
|
| 409 |
+
)
|
| 410 |
+
get_run_parser.add_argument(
|
| 411 |
+
"--json",
|
| 412 |
+
action="store_true",
|
| 413 |
+
help="Output in JSON format",
|
| 414 |
+
)
|
| 415 |
+
|
| 416 |
+
get_metric_parser = get_subparsers.add_parser(
|
| 417 |
+
"metric",
|
| 418 |
+
help="Get metric values for a run",
|
| 419 |
+
)
|
| 420 |
+
get_metric_parser.add_argument(
|
| 421 |
+
"--project",
|
| 422 |
+
required=True,
|
| 423 |
+
help="Project name",
|
| 424 |
+
)
|
| 425 |
+
get_metric_parser.add_argument(
|
| 426 |
+
"--run",
|
| 427 |
+
required=True,
|
| 428 |
+
help="Run name",
|
| 429 |
+
)
|
| 430 |
+
get_metric_parser.add_argument(
|
| 431 |
+
"--metric",
|
| 432 |
+
required=True,
|
| 433 |
+
help="Metric name",
|
| 434 |
+
)
|
| 435 |
+
get_metric_parser.add_argument(
|
| 436 |
+
"--step",
|
| 437 |
+
type=int,
|
| 438 |
+
required=False,
|
| 439 |
+
help="Get metric at exactly this step",
|
| 440 |
+
)
|
| 441 |
+
get_metric_parser.add_argument(
|
| 442 |
+
"--around",
|
| 443 |
+
type=int,
|
| 444 |
+
required=False,
|
| 445 |
+
help="Get metrics around this step (use with --window)",
|
| 446 |
+
)
|
| 447 |
+
get_metric_parser.add_argument(
|
| 448 |
+
"--at-time",
|
| 449 |
+
required=False,
|
| 450 |
+
help="Get metrics around this ISO 8601 timestamp (use with --window)",
|
| 451 |
+
)
|
| 452 |
+
get_metric_parser.add_argument(
|
| 453 |
+
"--window",
|
| 454 |
+
type=int,
|
| 455 |
+
required=False,
|
| 456 |
+
default=10,
|
| 457 |
+
help="Window size: ±steps for --around, ±seconds for --at-time (default: 10)",
|
| 458 |
+
)
|
| 459 |
+
get_metric_parser.add_argument(
|
| 460 |
+
"--json",
|
| 461 |
+
action="store_true",
|
| 462 |
+
help="Output in JSON format",
|
| 463 |
+
)
|
| 464 |
+
|
| 465 |
+
get_snapshot_parser = get_subparsers.add_parser(
|
| 466 |
+
"snapshot",
|
| 467 |
+
help="Get all metrics at/around a step or timestamp",
|
| 468 |
+
)
|
| 469 |
+
get_snapshot_parser.add_argument(
|
| 470 |
+
"--project",
|
| 471 |
+
required=True,
|
| 472 |
+
help="Project name",
|
| 473 |
+
)
|
| 474 |
+
get_snapshot_parser.add_argument(
|
| 475 |
+
"--run",
|
| 476 |
+
required=True,
|
| 477 |
+
help="Run name",
|
| 478 |
+
)
|
| 479 |
+
get_snapshot_parser.add_argument(
|
| 480 |
+
"--step",
|
| 481 |
+
type=int,
|
| 482 |
+
required=False,
|
| 483 |
+
help="Get all metrics at exactly this step",
|
| 484 |
+
)
|
| 485 |
+
get_snapshot_parser.add_argument(
|
| 486 |
+
"--around",
|
| 487 |
+
type=int,
|
| 488 |
+
required=False,
|
| 489 |
+
help="Get all metrics around this step (use with --window)",
|
| 490 |
+
)
|
| 491 |
+
get_snapshot_parser.add_argument(
|
| 492 |
+
"--at-time",
|
| 493 |
+
required=False,
|
| 494 |
+
help="Get all metrics around this ISO 8601 timestamp (use with --window)",
|
| 495 |
+
)
|
| 496 |
+
get_snapshot_parser.add_argument(
|
| 497 |
+
"--window",
|
| 498 |
+
type=int,
|
| 499 |
+
required=False,
|
| 500 |
+
default=10,
|
| 501 |
+
help="Window size: ±steps for --around, ±seconds for --at-time (default: 10)",
|
| 502 |
+
)
|
| 503 |
+
get_snapshot_parser.add_argument(
|
| 504 |
+
"--json",
|
| 505 |
+
action="store_true",
|
| 506 |
+
help="Output in JSON format",
|
| 507 |
+
)
|
| 508 |
+
|
| 509 |
+
get_system_metric_parser = get_subparsers.add_parser(
|
| 510 |
+
"system-metric",
|
| 511 |
+
help="Get system metric values for a run",
|
| 512 |
+
)
|
| 513 |
+
get_system_metric_parser.add_argument(
|
| 514 |
+
"--project",
|
| 515 |
+
required=True,
|
| 516 |
+
help="Project name",
|
| 517 |
+
)
|
| 518 |
+
get_system_metric_parser.add_argument(
|
| 519 |
+
"--run",
|
| 520 |
+
required=True,
|
| 521 |
+
help="Run name",
|
| 522 |
+
)
|
| 523 |
+
get_system_metric_parser.add_argument(
|
| 524 |
+
"--metric",
|
| 525 |
+
required=False,
|
| 526 |
+
help="System metric name (optional, if not provided returns all system metrics)",
|
| 527 |
+
)
|
| 528 |
+
get_system_metric_parser.add_argument(
|
| 529 |
+
"--json",
|
| 530 |
+
action="store_true",
|
| 531 |
+
help="Output in JSON format",
|
| 532 |
+
)
|
| 533 |
+
|
| 534 |
+
get_alerts_parser = get_subparsers.add_parser(
|
| 535 |
+
"alerts",
|
| 536 |
+
help="Get alerts for a project or run",
|
| 537 |
+
)
|
| 538 |
+
get_alerts_parser.add_argument(
|
| 539 |
+
"--project",
|
| 540 |
+
required=True,
|
| 541 |
+
help="Project name",
|
| 542 |
+
)
|
| 543 |
+
get_alerts_parser.add_argument(
|
| 544 |
+
"--run",
|
| 545 |
+
required=False,
|
| 546 |
+
help="Run name (optional)",
|
| 547 |
+
)
|
| 548 |
+
get_alerts_parser.add_argument(
|
| 549 |
+
"--level",
|
| 550 |
+
required=False,
|
| 551 |
+
help="Filter by alert level (info, warn, error)",
|
| 552 |
+
)
|
| 553 |
+
get_alerts_parser.add_argument(
|
| 554 |
+
"--json",
|
| 555 |
+
action="store_true",
|
| 556 |
+
help="Output in JSON format",
|
| 557 |
+
)
|
| 558 |
+
get_alerts_parser.add_argument(
|
| 559 |
+
"--since",
|
| 560 |
+
required=False,
|
| 561 |
+
help="Only show alerts after this ISO 8601 timestamp",
|
| 562 |
+
)
|
| 563 |
+
|
| 564 |
+
get_report_parser = get_subparsers.add_parser(
|
| 565 |
+
"report",
|
| 566 |
+
help="Get markdown report entries for a run",
|
| 567 |
+
)
|
| 568 |
+
get_report_parser.add_argument(
|
| 569 |
+
"--project",
|
| 570 |
+
required=True,
|
| 571 |
+
help="Project name",
|
| 572 |
+
)
|
| 573 |
+
get_report_parser.add_argument(
|
| 574 |
+
"--run",
|
| 575 |
+
required=True,
|
| 576 |
+
help="Run name",
|
| 577 |
+
)
|
| 578 |
+
get_report_parser.add_argument(
|
| 579 |
+
"--report",
|
| 580 |
+
required=True,
|
| 581 |
+
help="Report metric name",
|
| 582 |
+
)
|
| 583 |
+
get_report_parser.add_argument(
|
| 584 |
+
"--json",
|
| 585 |
+
action="store_true",
|
| 586 |
+
help="Output in JSON format",
|
| 587 |
+
)
|
| 588 |
+
|
| 589 |
+
skills_parser = subparsers.add_parser(
|
| 590 |
+
"skills",
|
| 591 |
+
help="Manage Trackio skills for AI coding assistants",
|
| 592 |
+
)
|
| 593 |
+
skills_subparsers = skills_parser.add_subparsers(
|
| 594 |
+
dest="skills_action", required=True
|
| 595 |
+
)
|
| 596 |
+
skills_add_parser = skills_subparsers.add_parser(
|
| 597 |
+
"add",
|
| 598 |
+
help="Download and install the Trackio skill for an AI assistant",
|
| 599 |
+
)
|
| 600 |
+
skills_add_parser.add_argument(
|
| 601 |
+
"--cursor",
|
| 602 |
+
action="store_true",
|
| 603 |
+
help="Install for Cursor",
|
| 604 |
+
)
|
| 605 |
+
skills_add_parser.add_argument(
|
| 606 |
+
"--claude",
|
| 607 |
+
action="store_true",
|
| 608 |
+
help="Install for Claude Code",
|
| 609 |
+
)
|
| 610 |
+
skills_add_parser.add_argument(
|
| 611 |
+
"--codex",
|
| 612 |
+
action="store_true",
|
| 613 |
+
help="Install for Codex",
|
| 614 |
+
)
|
| 615 |
+
skills_add_parser.add_argument(
|
| 616 |
+
"--opencode",
|
| 617 |
+
action="store_true",
|
| 618 |
+
help="Install for OpenCode",
|
| 619 |
+
)
|
| 620 |
+
skills_add_parser.add_argument(
|
| 621 |
+
"--global",
|
| 622 |
+
dest="global_",
|
| 623 |
+
action="store_true",
|
| 624 |
+
help="Install globally (user-level) instead of in the current project directory",
|
| 625 |
+
)
|
| 626 |
+
skills_add_parser.add_argument(
|
| 627 |
+
"--dest",
|
| 628 |
+
type=str,
|
| 629 |
+
required=False,
|
| 630 |
+
help="Install into a custom destination (path to skills directory)",
|
| 631 |
+
)
|
| 632 |
+
skills_add_parser.add_argument(
|
| 633 |
+
"--force",
|
| 634 |
+
action="store_true",
|
| 635 |
+
help="Overwrite existing skill if it already exists",
|
| 636 |
+
)
|
| 637 |
+
|
| 638 |
+
args, unknown_args = parser.parse_known_args()
|
| 639 |
+
if unknown_args:
|
| 640 |
+
trailing_global_parser = argparse.ArgumentParser(add_help=False)
|
| 641 |
+
trailing_global_parser.add_argument("--space", required=False)
|
| 642 |
+
trailing_global_parser.add_argument("--hf-token", required=False)
|
| 643 |
+
trailing_globals, remaining_unknown = trailing_global_parser.parse_known_args(
|
| 644 |
+
unknown_args
|
| 645 |
+
)
|
| 646 |
+
if remaining_unknown:
|
| 647 |
+
parser.error(f"unrecognized arguments: {' '.join(remaining_unknown)}")
|
| 648 |
+
if trailing_globals.space is not None:
|
| 649 |
+
args.space = trailing_globals.space
|
| 650 |
+
if trailing_globals.hf_token is not None:
|
| 651 |
+
args.hf_token = trailing_globals.hf_token
|
| 652 |
+
|
| 653 |
+
if args.command in ("show", "status", "sync", "freeze", "skills") and _get_space(
|
| 654 |
+
args
|
| 655 |
+
):
|
| 656 |
+
error_exit(
|
| 657 |
+
f"The '{args.command}' command does not support --space (remote mode)."
|
| 658 |
+
)
|
| 659 |
+
|
| 660 |
+
if args.command == "show":
|
| 661 |
+
color_palette = None
|
| 662 |
+
if args.color_palette:
|
| 663 |
+
color_palette = [color.strip() for color in args.color_palette.split(",")]
|
| 664 |
+
show(
|
| 665 |
+
project=args.project,
|
| 666 |
+
theme=args.theme,
|
| 667 |
+
mcp_server=args.mcp_server,
|
| 668 |
+
footer=args.footer,
|
| 669 |
+
color_palette=color_palette,
|
| 670 |
+
host=args.host,
|
| 671 |
+
)
|
| 672 |
+
elif args.command == "status":
|
| 673 |
+
_handle_status()
|
| 674 |
+
elif args.command == "sync":
|
| 675 |
+
_handle_sync(args)
|
| 676 |
+
elif args.command == "freeze":
|
| 677 |
+
freeze(
|
| 678 |
+
space_id=args.space_id,
|
| 679 |
+
project=args.project,
|
| 680 |
+
new_space_id=args.new_space_id,
|
| 681 |
+
private=args.private,
|
| 682 |
+
)
|
| 683 |
+
elif args.command == "list":
|
| 684 |
+
remote = _get_remote(args)
|
| 685 |
+
if args.list_type == "projects":
|
| 686 |
+
if remote:
|
| 687 |
+
projects = remote.predict(api_name="/get_all_projects")
|
| 688 |
+
else:
|
| 689 |
+
projects = SQLiteStorage.get_projects()
|
| 690 |
+
if args.json:
|
| 691 |
+
print(format_json({"projects": projects}))
|
| 692 |
+
else:
|
| 693 |
+
print(format_list(projects, "Projects"))
|
| 694 |
+
elif args.list_type == "runs":
|
| 695 |
+
if remote:
|
| 696 |
+
runs = remote.predict(args.project, api_name="/get_runs_for_project")
|
| 697 |
+
else:
|
| 698 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 699 |
+
if not db_path.exists():
|
| 700 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 701 |
+
runs = SQLiteStorage.get_runs(args.project)
|
| 702 |
+
if args.json:
|
| 703 |
+
print(format_json({"project": args.project, "runs": runs}))
|
| 704 |
+
else:
|
| 705 |
+
print(format_list(runs, f"Runs in '{args.project}'"))
|
| 706 |
+
elif args.list_type == "metrics":
|
| 707 |
+
if remote:
|
| 708 |
+
metrics = remote.predict(
|
| 709 |
+
args.project, args.run, api_name="/get_metrics_for_run"
|
| 710 |
+
)
|
| 711 |
+
else:
|
| 712 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 713 |
+
if not db_path.exists():
|
| 714 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 715 |
+
runs = SQLiteStorage.get_runs(args.project)
|
| 716 |
+
if args.run not in runs:
|
| 717 |
+
error_exit(
|
| 718 |
+
f"Run '{args.run}' not found in project '{args.project}'."
|
| 719 |
+
)
|
| 720 |
+
metrics = SQLiteStorage.get_all_metrics_for_run(args.project, args.run)
|
| 721 |
+
if args.json:
|
| 722 |
+
print(
|
| 723 |
+
format_json(
|
| 724 |
+
{"project": args.project, "run": args.run, "metrics": metrics}
|
| 725 |
+
)
|
| 726 |
+
)
|
| 727 |
+
else:
|
| 728 |
+
print(
|
| 729 |
+
format_list(
|
| 730 |
+
metrics, f"Metrics for '{args.run}' in '{args.project}'"
|
| 731 |
+
)
|
| 732 |
+
)
|
| 733 |
+
elif args.list_type == "system-metrics":
|
| 734 |
+
if remote:
|
| 735 |
+
system_metrics = remote.predict(
|
| 736 |
+
args.project, args.run, api_name="/get_system_metrics_for_run"
|
| 737 |
+
)
|
| 738 |
+
else:
|
| 739 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 740 |
+
if not db_path.exists():
|
| 741 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 742 |
+
runs = SQLiteStorage.get_runs(args.project)
|
| 743 |
+
if args.run not in runs:
|
| 744 |
+
error_exit(
|
| 745 |
+
f"Run '{args.run}' not found in project '{args.project}'."
|
| 746 |
+
)
|
| 747 |
+
system_metrics = SQLiteStorage.get_all_system_metrics_for_run(
|
| 748 |
+
args.project, args.run
|
| 749 |
+
)
|
| 750 |
+
if args.json:
|
| 751 |
+
print(
|
| 752 |
+
format_json(
|
| 753 |
+
{
|
| 754 |
+
"project": args.project,
|
| 755 |
+
"run": args.run,
|
| 756 |
+
"system_metrics": system_metrics,
|
| 757 |
+
}
|
| 758 |
+
)
|
| 759 |
+
)
|
| 760 |
+
else:
|
| 761 |
+
print(format_system_metric_names(system_metrics))
|
| 762 |
+
elif args.list_type == "alerts":
|
| 763 |
+
if remote:
|
| 764 |
+
alerts = remote.predict(
|
| 765 |
+
args.project,
|
| 766 |
+
args.run,
|
| 767 |
+
args.level,
|
| 768 |
+
args.since,
|
| 769 |
+
api_name="/get_alerts",
|
| 770 |
+
)
|
| 771 |
+
else:
|
| 772 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 773 |
+
if not db_path.exists():
|
| 774 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 775 |
+
alerts = SQLiteStorage.get_alerts(
|
| 776 |
+
args.project,
|
| 777 |
+
run_name=args.run,
|
| 778 |
+
level=args.level,
|
| 779 |
+
since=args.since,
|
| 780 |
+
)
|
| 781 |
+
if args.json:
|
| 782 |
+
print(
|
| 783 |
+
format_json(
|
| 784 |
+
{
|
| 785 |
+
"project": args.project,
|
| 786 |
+
"run": args.run,
|
| 787 |
+
"level": args.level,
|
| 788 |
+
"since": args.since,
|
| 789 |
+
"alerts": alerts,
|
| 790 |
+
}
|
| 791 |
+
)
|
| 792 |
+
)
|
| 793 |
+
else:
|
| 794 |
+
print(format_alerts(alerts))
|
| 795 |
+
elif args.list_type == "reports":
|
| 796 |
+
if remote:
|
| 797 |
+
runs = remote.predict(args.project, api_name="/get_runs_for_project")
|
| 798 |
+
else:
|
| 799 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 800 |
+
if not db_path.exists():
|
| 801 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 802 |
+
runs = SQLiteStorage.get_runs(args.project)
|
| 803 |
+
if args.run and args.run not in runs:
|
| 804 |
+
error_exit(f"Run '{args.run}' not found in project '{args.project}'.")
|
| 805 |
+
|
| 806 |
+
target_runs = [args.run] if args.run else runs
|
| 807 |
+
all_reports = []
|
| 808 |
+
for run_name in target_runs:
|
| 809 |
+
if remote:
|
| 810 |
+
logs = remote.predict(args.project, run_name, api_name="/get_logs")
|
| 811 |
+
else:
|
| 812 |
+
logs = SQLiteStorage.get_logs(args.project, run_name)
|
| 813 |
+
all_reports.extend(_extract_reports(run_name, logs))
|
| 814 |
+
|
| 815 |
+
if args.json:
|
| 816 |
+
print(
|
| 817 |
+
format_json(
|
| 818 |
+
{
|
| 819 |
+
"project": args.project,
|
| 820 |
+
"run": args.run,
|
| 821 |
+
"reports": all_reports,
|
| 822 |
+
}
|
| 823 |
+
)
|
| 824 |
+
)
|
| 825 |
+
else:
|
| 826 |
+
report_lines = [
|
| 827 |
+
f"{entry['run']} | {entry['report']} | step={entry['step']} | {entry['timestamp']}"
|
| 828 |
+
for entry in all_reports
|
| 829 |
+
]
|
| 830 |
+
if args.run:
|
| 831 |
+
print(
|
| 832 |
+
format_list(
|
| 833 |
+
report_lines,
|
| 834 |
+
f"Reports for '{args.run}' in '{args.project}'",
|
| 835 |
+
)
|
| 836 |
+
)
|
| 837 |
+
else:
|
| 838 |
+
print(format_list(report_lines, f"Reports in '{args.project}'"))
|
| 839 |
+
elif args.command == "get":
|
| 840 |
+
remote = _get_remote(args)
|
| 841 |
+
if args.get_type == "project":
|
| 842 |
+
if remote:
|
| 843 |
+
summary = remote.predict(args.project, api_name="/get_project_summary")
|
| 844 |
+
else:
|
| 845 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 846 |
+
if not db_path.exists():
|
| 847 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 848 |
+
summary = get_project_summary(args.project)
|
| 849 |
+
if args.json:
|
| 850 |
+
print(format_json(summary))
|
| 851 |
+
else:
|
| 852 |
+
print(format_project_summary(summary))
|
| 853 |
+
elif args.get_type == "run":
|
| 854 |
+
if remote:
|
| 855 |
+
summary = remote.predict(
|
| 856 |
+
args.project, args.run, api_name="/get_run_summary"
|
| 857 |
+
)
|
| 858 |
+
else:
|
| 859 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 860 |
+
if not db_path.exists():
|
| 861 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 862 |
+
runs = SQLiteStorage.get_runs(args.project)
|
| 863 |
+
if args.run not in runs:
|
| 864 |
+
error_exit(
|
| 865 |
+
f"Run '{args.run}' not found in project '{args.project}'."
|
| 866 |
+
)
|
| 867 |
+
summary = get_run_summary(args.project, args.run)
|
| 868 |
+
if args.json:
|
| 869 |
+
print(format_json(summary))
|
| 870 |
+
else:
|
| 871 |
+
print(format_run_summary(summary))
|
| 872 |
+
elif args.get_type == "metric":
|
| 873 |
+
at_time = getattr(args, "at_time", None)
|
| 874 |
+
if remote:
|
| 875 |
+
values = remote.predict(
|
| 876 |
+
args.project,
|
| 877 |
+
args.run,
|
| 878 |
+
args.metric,
|
| 879 |
+
args.step,
|
| 880 |
+
args.around,
|
| 881 |
+
at_time,
|
| 882 |
+
args.window,
|
| 883 |
+
api_name="/get_metric_values",
|
| 884 |
+
)
|
| 885 |
+
else:
|
| 886 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 887 |
+
if not db_path.exists():
|
| 888 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 889 |
+
runs = SQLiteStorage.get_runs(args.project)
|
| 890 |
+
if args.run not in runs:
|
| 891 |
+
error_exit(
|
| 892 |
+
f"Run '{args.run}' not found in project '{args.project}'."
|
| 893 |
+
)
|
| 894 |
+
metrics = SQLiteStorage.get_all_metrics_for_run(args.project, args.run)
|
| 895 |
+
if args.metric not in metrics:
|
| 896 |
+
error_exit(
|
| 897 |
+
f"Metric '{args.metric}' not found in run '{args.run}' of project '{args.project}'."
|
| 898 |
+
)
|
| 899 |
+
values = SQLiteStorage.get_metric_values(
|
| 900 |
+
args.project,
|
| 901 |
+
args.run,
|
| 902 |
+
args.metric,
|
| 903 |
+
step=args.step,
|
| 904 |
+
around_step=args.around,
|
| 905 |
+
at_time=at_time,
|
| 906 |
+
window=args.window,
|
| 907 |
+
)
|
| 908 |
+
if args.json:
|
| 909 |
+
print(
|
| 910 |
+
format_json(
|
| 911 |
+
{
|
| 912 |
+
"project": args.project,
|
| 913 |
+
"run": args.run,
|
| 914 |
+
"metric": args.metric,
|
| 915 |
+
"values": values,
|
| 916 |
+
}
|
| 917 |
+
)
|
| 918 |
+
)
|
| 919 |
+
else:
|
| 920 |
+
print(format_metric_values(values))
|
| 921 |
+
elif args.get_type == "snapshot":
|
| 922 |
+
if not args.step and not args.around and not getattr(args, "at_time", None):
|
| 923 |
+
error_exit(
|
| 924 |
+
"Provide --step, --around (with --window), or --at-time (with --window)."
|
| 925 |
+
)
|
| 926 |
+
at_time = getattr(args, "at_time", None)
|
| 927 |
+
if remote:
|
| 928 |
+
snapshot = remote.predict(
|
| 929 |
+
args.project,
|
| 930 |
+
args.run,
|
| 931 |
+
args.step,
|
| 932 |
+
args.around,
|
| 933 |
+
at_time,
|
| 934 |
+
args.window,
|
| 935 |
+
api_name="/get_snapshot",
|
| 936 |
+
)
|
| 937 |
+
else:
|
| 938 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 939 |
+
if not db_path.exists():
|
| 940 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 941 |
+
runs = SQLiteStorage.get_runs(args.project)
|
| 942 |
+
if args.run not in runs:
|
| 943 |
+
error_exit(
|
| 944 |
+
f"Run '{args.run}' not found in project '{args.project}'."
|
| 945 |
+
)
|
| 946 |
+
snapshot = SQLiteStorage.get_snapshot(
|
| 947 |
+
args.project,
|
| 948 |
+
args.run,
|
| 949 |
+
step=args.step,
|
| 950 |
+
around_step=args.around,
|
| 951 |
+
at_time=at_time,
|
| 952 |
+
window=args.window,
|
| 953 |
+
)
|
| 954 |
+
if args.json:
|
| 955 |
+
result = {
|
| 956 |
+
"project": args.project,
|
| 957 |
+
"run": args.run,
|
| 958 |
+
"metrics": snapshot,
|
| 959 |
+
}
|
| 960 |
+
if args.step is not None:
|
| 961 |
+
result["step"] = args.step
|
| 962 |
+
if args.around is not None:
|
| 963 |
+
result["around"] = args.around
|
| 964 |
+
result["window"] = args.window
|
| 965 |
+
if at_time is not None:
|
| 966 |
+
result["at_time"] = at_time
|
| 967 |
+
result["window"] = args.window
|
| 968 |
+
print(format_json(result))
|
| 969 |
+
else:
|
| 970 |
+
print(format_snapshot(snapshot))
|
| 971 |
+
elif args.get_type == "system-metric":
|
| 972 |
+
if remote:
|
| 973 |
+
system_metrics = remote.predict(
|
| 974 |
+
args.project, args.run, api_name="/get_system_logs"
|
| 975 |
+
)
|
| 976 |
+
if args.metric:
|
| 977 |
+
all_system_metric_names = remote.predict(
|
| 978 |
+
args.project,
|
| 979 |
+
args.run,
|
| 980 |
+
api_name="/get_system_metrics_for_run",
|
| 981 |
+
)
|
| 982 |
+
if args.metric not in all_system_metric_names:
|
| 983 |
+
error_exit(
|
| 984 |
+
f"System metric '{args.metric}' not found in run '{args.run}' of project '{args.project}'."
|
| 985 |
+
)
|
| 986 |
+
filtered_metrics = [
|
| 987 |
+
{
|
| 988 |
+
k: v
|
| 989 |
+
for k, v in entry.items()
|
| 990 |
+
if k == "timestamp" or k == args.metric
|
| 991 |
+
}
|
| 992 |
+
for entry in system_metrics
|
| 993 |
+
if args.metric in entry
|
| 994 |
+
]
|
| 995 |
+
if args.json:
|
| 996 |
+
print(
|
| 997 |
+
format_json(
|
| 998 |
+
{
|
| 999 |
+
"project": args.project,
|
| 1000 |
+
"run": args.run,
|
| 1001 |
+
"metric": args.metric,
|
| 1002 |
+
"values": filtered_metrics,
|
| 1003 |
+
}
|
| 1004 |
+
)
|
| 1005 |
+
)
|
| 1006 |
+
else:
|
| 1007 |
+
print(format_system_metrics(filtered_metrics))
|
| 1008 |
+
else:
|
| 1009 |
+
if args.json:
|
| 1010 |
+
print(
|
| 1011 |
+
format_json(
|
| 1012 |
+
{
|
| 1013 |
+
"project": args.project,
|
| 1014 |
+
"run": args.run,
|
| 1015 |
+
"system_metrics": system_metrics,
|
| 1016 |
+
}
|
| 1017 |
+
)
|
| 1018 |
+
)
|
| 1019 |
+
else:
|
| 1020 |
+
print(format_system_metrics(system_metrics))
|
| 1021 |
+
else:
|
| 1022 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 1023 |
+
if not db_path.exists():
|
| 1024 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 1025 |
+
runs = SQLiteStorage.get_runs(args.project)
|
| 1026 |
+
if args.run not in runs:
|
| 1027 |
+
error_exit(
|
| 1028 |
+
f"Run '{args.run}' not found in project '{args.project}'."
|
| 1029 |
+
)
|
| 1030 |
+
if args.metric:
|
| 1031 |
+
system_metrics = SQLiteStorage.get_system_logs(
|
| 1032 |
+
args.project, args.run
|
| 1033 |
+
)
|
| 1034 |
+
all_system_metric_names = (
|
| 1035 |
+
SQLiteStorage.get_all_system_metrics_for_run(
|
| 1036 |
+
args.project, args.run
|
| 1037 |
+
)
|
| 1038 |
+
)
|
| 1039 |
+
if args.metric not in all_system_metric_names:
|
| 1040 |
+
error_exit(
|
| 1041 |
+
f"System metric '{args.metric}' not found in run '{args.run}' of project '{args.project}'."
|
| 1042 |
+
)
|
| 1043 |
+
filtered_metrics = [
|
| 1044 |
+
{
|
| 1045 |
+
k: v
|
| 1046 |
+
for k, v in entry.items()
|
| 1047 |
+
if k == "timestamp" or k == args.metric
|
| 1048 |
+
}
|
| 1049 |
+
for entry in system_metrics
|
| 1050 |
+
if args.metric in entry
|
| 1051 |
+
]
|
| 1052 |
+
if args.json:
|
| 1053 |
+
print(
|
| 1054 |
+
format_json(
|
| 1055 |
+
{
|
| 1056 |
+
"project": args.project,
|
| 1057 |
+
"run": args.run,
|
| 1058 |
+
"metric": args.metric,
|
| 1059 |
+
"values": filtered_metrics,
|
| 1060 |
+
}
|
| 1061 |
+
)
|
| 1062 |
+
)
|
| 1063 |
+
else:
|
| 1064 |
+
print(format_system_metrics(filtered_metrics))
|
| 1065 |
+
else:
|
| 1066 |
+
system_metrics = SQLiteStorage.get_system_logs(
|
| 1067 |
+
args.project, args.run
|
| 1068 |
+
)
|
| 1069 |
+
if args.json:
|
| 1070 |
+
print(
|
| 1071 |
+
format_json(
|
| 1072 |
+
{
|
| 1073 |
+
"project": args.project,
|
| 1074 |
+
"run": args.run,
|
| 1075 |
+
"system_metrics": system_metrics,
|
| 1076 |
+
}
|
| 1077 |
+
)
|
| 1078 |
+
)
|
| 1079 |
+
else:
|
| 1080 |
+
print(format_system_metrics(system_metrics))
|
| 1081 |
+
elif args.get_type == "alerts":
|
| 1082 |
+
if remote:
|
| 1083 |
+
alerts = remote.predict(
|
| 1084 |
+
args.project,
|
| 1085 |
+
args.run,
|
| 1086 |
+
args.level,
|
| 1087 |
+
args.since,
|
| 1088 |
+
api_name="/get_alerts",
|
| 1089 |
+
)
|
| 1090 |
+
else:
|
| 1091 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 1092 |
+
if not db_path.exists():
|
| 1093 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 1094 |
+
alerts = SQLiteStorage.get_alerts(
|
| 1095 |
+
args.project,
|
| 1096 |
+
run_name=args.run,
|
| 1097 |
+
level=args.level,
|
| 1098 |
+
since=args.since,
|
| 1099 |
+
)
|
| 1100 |
+
if args.json:
|
| 1101 |
+
print(
|
| 1102 |
+
format_json(
|
| 1103 |
+
{
|
| 1104 |
+
"project": args.project,
|
| 1105 |
+
"run": args.run,
|
| 1106 |
+
"level": args.level,
|
| 1107 |
+
"since": args.since,
|
| 1108 |
+
"alerts": alerts,
|
| 1109 |
+
}
|
| 1110 |
+
)
|
| 1111 |
+
)
|
| 1112 |
+
else:
|
| 1113 |
+
print(format_alerts(alerts))
|
| 1114 |
+
elif args.get_type == "report":
|
| 1115 |
+
if remote:
|
| 1116 |
+
logs = remote.predict(args.project, args.run, api_name="/get_logs")
|
| 1117 |
+
else:
|
| 1118 |
+
db_path = SQLiteStorage.get_project_db_path(args.project)
|
| 1119 |
+
if not db_path.exists():
|
| 1120 |
+
error_exit(f"Project '{args.project}' not found.")
|
| 1121 |
+
runs = SQLiteStorage.get_runs(args.project)
|
| 1122 |
+
if args.run not in runs:
|
| 1123 |
+
error_exit(
|
| 1124 |
+
f"Run '{args.run}' not found in project '{args.project}'."
|
| 1125 |
+
)
|
| 1126 |
+
logs = SQLiteStorage.get_logs(args.project, args.run)
|
| 1127 |
+
|
| 1128 |
+
reports = _extract_reports(args.run, logs, report_name=args.report)
|
| 1129 |
+
if not reports:
|
| 1130 |
+
error_exit(
|
| 1131 |
+
f"Report '{args.report}' not found in run '{args.run}' of project '{args.project}'."
|
| 1132 |
+
)
|
| 1133 |
+
|
| 1134 |
+
if args.json:
|
| 1135 |
+
print(
|
| 1136 |
+
format_json(
|
| 1137 |
+
{
|
| 1138 |
+
"project": args.project,
|
| 1139 |
+
"run": args.run,
|
| 1140 |
+
"report": args.report,
|
| 1141 |
+
"values": reports,
|
| 1142 |
+
}
|
| 1143 |
+
)
|
| 1144 |
+
)
|
| 1145 |
+
else:
|
| 1146 |
+
output = []
|
| 1147 |
+
for idx, entry in enumerate(reports, start=1):
|
| 1148 |
+
output.append(
|
| 1149 |
+
f"Entry {idx} | step={entry['step']} | timestamp={entry['timestamp']}"
|
| 1150 |
+
)
|
| 1151 |
+
output.append(entry["content"])
|
| 1152 |
+
if idx < len(reports):
|
| 1153 |
+
output.append("-" * 80)
|
| 1154 |
+
print("\n".join(output))
|
| 1155 |
+
elif args.command == "skills":
|
| 1156 |
+
if args.skills_action == "add":
|
| 1157 |
+
_handle_skills_add(args)
|
| 1158 |
+
else:
|
| 1159 |
+
parser.print_help()
|
| 1160 |
+
|
| 1161 |
+
|
| 1162 |
+
def _handle_skills_add(args):
|
| 1163 |
+
import shutil
|
| 1164 |
+
from pathlib import Path
|
| 1165 |
+
|
| 1166 |
+
try:
|
| 1167 |
+
from huggingface_hub.cli.skills import (
|
| 1168 |
+
CENTRAL_GLOBAL,
|
| 1169 |
+
CENTRAL_LOCAL,
|
| 1170 |
+
GLOBAL_TARGETS,
|
| 1171 |
+
LOCAL_TARGETS,
|
| 1172 |
+
)
|
| 1173 |
+
except (ImportError, ModuleNotFoundError):
|
| 1174 |
+
error_exit(
|
| 1175 |
+
"The 'trackio skills' command requires huggingface_hub >= 1.4.0.\n"
|
| 1176 |
+
"Please upgrade: pip install --upgrade huggingface_hub"
|
| 1177 |
+
)
|
| 1178 |
+
|
| 1179 |
+
SKILL_ID = "trackio"
|
| 1180 |
+
GITHUB_RAW = "https://raw.githubusercontent.com/gradio-app/trackio/main"
|
| 1181 |
+
SKILL_PREFIX = ".agents/skills/trackio"
|
| 1182 |
+
SKILL_FILES = [
|
| 1183 |
+
"SKILL.md",
|
| 1184 |
+
"alerts.md",
|
| 1185 |
+
"logging_metrics.md",
|
| 1186 |
+
"retrieving_metrics.md",
|
| 1187 |
+
]
|
| 1188 |
+
|
| 1189 |
+
if not (args.cursor or args.claude or args.codex or args.opencode or args.dest):
|
| 1190 |
+
error_exit(
|
| 1191 |
+
"Pick a destination via --cursor, --claude, --codex, --opencode, or --dest."
|
| 1192 |
+
)
|
| 1193 |
+
|
| 1194 |
+
def download(url: str) -> str:
|
| 1195 |
+
from huggingface_hub.utils import get_session
|
| 1196 |
+
|
| 1197 |
+
try:
|
| 1198 |
+
response = get_session().get(url)
|
| 1199 |
+
response.raise_for_status()
|
| 1200 |
+
except Exception as e:
|
| 1201 |
+
error_exit(
|
| 1202 |
+
f"Failed to download {url}\n{e}\n\n"
|
| 1203 |
+
"Make sure you have internet access. The skill files are fetched from "
|
| 1204 |
+
"the Trackio GitHub repository."
|
| 1205 |
+
)
|
| 1206 |
+
return response.text
|
| 1207 |
+
|
| 1208 |
+
def remove_existing(path: Path, force: bool):
|
| 1209 |
+
if not (path.exists() or path.is_symlink()):
|
| 1210 |
+
return
|
| 1211 |
+
if not force:
|
| 1212 |
+
error_exit(
|
| 1213 |
+
f"Skill already exists at {path}.\nRe-run with --force to overwrite."
|
| 1214 |
+
)
|
| 1215 |
+
if path.is_dir() and not path.is_symlink():
|
| 1216 |
+
shutil.rmtree(path)
|
| 1217 |
+
else:
|
| 1218 |
+
path.unlink()
|
| 1219 |
+
|
| 1220 |
+
def install_to(skills_dir: Path, force: bool) -> Path:
|
| 1221 |
+
skills_dir = skills_dir.expanduser().resolve()
|
| 1222 |
+
skills_dir.mkdir(parents=True, exist_ok=True)
|
| 1223 |
+
dest = skills_dir / SKILL_ID
|
| 1224 |
+
remove_existing(dest, force)
|
| 1225 |
+
dest.mkdir()
|
| 1226 |
+
for fname in SKILL_FILES:
|
| 1227 |
+
content = download(f"{GITHUB_RAW}/{SKILL_PREFIX}/{fname}")
|
| 1228 |
+
(dest / fname).write_text(content, encoding="utf-8")
|
| 1229 |
+
return dest
|
| 1230 |
+
|
| 1231 |
+
def create_symlink(
|
| 1232 |
+
agent_skills_dir: Path, central_skill_path: Path, force: bool
|
| 1233 |
+
) -> Path:
|
| 1234 |
+
agent_skills_dir = agent_skills_dir.expanduser().resolve()
|
| 1235 |
+
agent_skills_dir.mkdir(parents=True, exist_ok=True)
|
| 1236 |
+
link_path = agent_skills_dir / SKILL_ID
|
| 1237 |
+
remove_existing(link_path, force)
|
| 1238 |
+
link_path.symlink_to(os.path.relpath(central_skill_path, agent_skills_dir))
|
| 1239 |
+
return link_path
|
| 1240 |
+
|
| 1241 |
+
global_targets = {**GLOBAL_TARGETS, "cursor": Path("~/.cursor/skills")}
|
| 1242 |
+
local_targets = {**LOCAL_TARGETS, "cursor": Path(".cursor/skills")}
|
| 1243 |
+
targets_dict = global_targets if args.global_ else local_targets
|
| 1244 |
+
|
| 1245 |
+
if args.dest:
|
| 1246 |
+
if args.cursor or args.claude or args.codex or args.opencode or args.global_:
|
| 1247 |
+
error_exit("--dest cannot be combined with agent flags or --global.")
|
| 1248 |
+
skill_dest = install_to(Path(args.dest), args.force)
|
| 1249 |
+
print(f"Installed '{SKILL_ID}' to {skill_dest}")
|
| 1250 |
+
return
|
| 1251 |
+
|
| 1252 |
+
agent_targets = []
|
| 1253 |
+
if args.cursor:
|
| 1254 |
+
agent_targets.append(targets_dict["cursor"])
|
| 1255 |
+
if args.claude:
|
| 1256 |
+
agent_targets.append(targets_dict["claude"])
|
| 1257 |
+
if args.codex:
|
| 1258 |
+
agent_targets.append(targets_dict["codex"])
|
| 1259 |
+
if args.opencode:
|
| 1260 |
+
agent_targets.append(targets_dict["opencode"])
|
| 1261 |
+
|
| 1262 |
+
central_path = CENTRAL_GLOBAL if args.global_ else CENTRAL_LOCAL
|
| 1263 |
+
central_skill_path = install_to(central_path, args.force)
|
| 1264 |
+
print(f"Installed '{SKILL_ID}' to central location: {central_skill_path}")
|
| 1265 |
+
|
| 1266 |
+
for agent_target in agent_targets:
|
| 1267 |
+
link_path = create_symlink(agent_target, central_skill_path, args.force)
|
| 1268 |
+
print(f"Created symlink: {link_path}")
|
| 1269 |
+
|
| 1270 |
+
|
| 1271 |
+
if __name__ == "__main__":
|
| 1272 |
+
main()
|
trackio/cli_helpers.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import sys
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def format_json(data: Any) -> str:
|
| 7 |
+
"""Format data as JSON."""
|
| 8 |
+
return json.dumps(data, indent=2)
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def format_list(items: list[str], title: str | None = None) -> str:
|
| 12 |
+
"""Format a list of items in human-readable format."""
|
| 13 |
+
if not items:
|
| 14 |
+
return f"No {title.lower() if title else 'items'} found."
|
| 15 |
+
|
| 16 |
+
output = []
|
| 17 |
+
if title:
|
| 18 |
+
output.append(f"{title}:")
|
| 19 |
+
|
| 20 |
+
for item in items:
|
| 21 |
+
output.append(f" - {item}")
|
| 22 |
+
|
| 23 |
+
return "\n".join(output)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def format_project_summary(summary: dict) -> str:
|
| 27 |
+
"""Format project summary in human-readable format."""
|
| 28 |
+
output = [f"Project: {summary['project']}"]
|
| 29 |
+
output.append(f"Number of runs: {summary['num_runs']}")
|
| 30 |
+
|
| 31 |
+
if summary["runs"]:
|
| 32 |
+
output.append("\nRuns:")
|
| 33 |
+
for run in summary["runs"]:
|
| 34 |
+
output.append(f" - {run}")
|
| 35 |
+
else:
|
| 36 |
+
output.append("\nNo runs found.")
|
| 37 |
+
|
| 38 |
+
if summary.get("last_activity"):
|
| 39 |
+
output.append(f"\nLast activity (max step): {summary['last_activity']}")
|
| 40 |
+
|
| 41 |
+
return "\n".join(output)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def format_run_summary(summary: dict) -> str:
|
| 45 |
+
"""Format run summary in human-readable format."""
|
| 46 |
+
output = [f"Project: {summary['project']}"]
|
| 47 |
+
output.append(f"Run: {summary['run']}")
|
| 48 |
+
output.append(f"Number of logs: {summary['num_logs']}")
|
| 49 |
+
|
| 50 |
+
if summary.get("last_step") is not None:
|
| 51 |
+
output.append(f"Last step: {summary['last_step']}")
|
| 52 |
+
|
| 53 |
+
if summary.get("metrics"):
|
| 54 |
+
output.append("\nMetrics:")
|
| 55 |
+
for metric in summary["metrics"]:
|
| 56 |
+
output.append(f" - {metric}")
|
| 57 |
+
else:
|
| 58 |
+
output.append("\nNo metrics found.")
|
| 59 |
+
|
| 60 |
+
config = summary.get("config")
|
| 61 |
+
if config:
|
| 62 |
+
output.append("\nConfig:")
|
| 63 |
+
config_display = {k: v for k, v in config.items() if not k.startswith("_")}
|
| 64 |
+
if config_display:
|
| 65 |
+
for key, value in config_display.items():
|
| 66 |
+
output.append(f" {key}: {value}")
|
| 67 |
+
else:
|
| 68 |
+
output.append(" (no config)")
|
| 69 |
+
else:
|
| 70 |
+
output.append("\nConfig: (no config)")
|
| 71 |
+
|
| 72 |
+
return "\n".join(output)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def format_metric_values(values: list[dict]) -> str:
|
| 76 |
+
"""Format metric values in human-readable format."""
|
| 77 |
+
if not values:
|
| 78 |
+
return "No metric values found."
|
| 79 |
+
|
| 80 |
+
output = [f"Found {len(values)} value(s):\n"]
|
| 81 |
+
output.append("Step | Timestamp | Value")
|
| 82 |
+
output.append("-" * 50)
|
| 83 |
+
|
| 84 |
+
for value in values:
|
| 85 |
+
step = value.get("step", "N/A")
|
| 86 |
+
timestamp = value.get("timestamp", "N/A")
|
| 87 |
+
val = value.get("value", "N/A")
|
| 88 |
+
output.append(f"{step} | {timestamp} | {val}")
|
| 89 |
+
|
| 90 |
+
return "\n".join(output)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def format_system_metrics(metrics: list[dict]) -> str:
|
| 94 |
+
"""Format system metrics in human-readable format."""
|
| 95 |
+
if not metrics:
|
| 96 |
+
return "No system metrics found."
|
| 97 |
+
|
| 98 |
+
output = [f"Found {len(metrics)} system metric entry/entries:\n"]
|
| 99 |
+
|
| 100 |
+
for i, entry in enumerate(metrics):
|
| 101 |
+
timestamp = entry.get("timestamp", "N/A")
|
| 102 |
+
output.append(f"\nEntry {i + 1} (Timestamp: {timestamp}):")
|
| 103 |
+
for key, value in entry.items():
|
| 104 |
+
if key != "timestamp":
|
| 105 |
+
output.append(f" {key}: {value}")
|
| 106 |
+
|
| 107 |
+
return "\n".join(output)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def format_system_metric_names(names: list[str]) -> str:
|
| 111 |
+
"""Format system metric names in human-readable format."""
|
| 112 |
+
return format_list(names, "System Metrics")
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def format_snapshot(snapshot: dict[str, list[dict]]) -> str:
|
| 116 |
+
"""Format a metrics snapshot in human-readable format."""
|
| 117 |
+
if not snapshot:
|
| 118 |
+
return "No metrics found in the specified range."
|
| 119 |
+
|
| 120 |
+
output = []
|
| 121 |
+
for metric_name, values in sorted(snapshot.items()):
|
| 122 |
+
output.append(f"\n{metric_name}:")
|
| 123 |
+
output.append(" Step | Timestamp | Value")
|
| 124 |
+
output.append(" " + "-" * 48)
|
| 125 |
+
for v in values:
|
| 126 |
+
step = v.get("step", "N/A")
|
| 127 |
+
ts = v.get("timestamp", "N/A")
|
| 128 |
+
val = v.get("value", "N/A")
|
| 129 |
+
output.append(f" {step} | {ts} | {val}")
|
| 130 |
+
|
| 131 |
+
return "\n".join(output)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def format_alerts(alerts: list[dict]) -> str:
|
| 135 |
+
"""Format alerts in human-readable format."""
|
| 136 |
+
if not alerts:
|
| 137 |
+
return "No alerts found."
|
| 138 |
+
|
| 139 |
+
output = [f"Found {len(alerts)} alert(s):\n"]
|
| 140 |
+
output.append("Timestamp | Run | Level | Title | Text | Step")
|
| 141 |
+
output.append("-" * 80)
|
| 142 |
+
|
| 143 |
+
for a in alerts:
|
| 144 |
+
ts = a.get("timestamp", "N/A")
|
| 145 |
+
run = a.get("run", "N/A")
|
| 146 |
+
level = a.get("level", "N/A").upper()
|
| 147 |
+
title = a.get("title", "")
|
| 148 |
+
text = a.get("text", "") or ""
|
| 149 |
+
step = a.get("step", "N/A")
|
| 150 |
+
output.append(f"{ts} | {run} | {level} | {title} | {text} | {step}")
|
| 151 |
+
|
| 152 |
+
return "\n".join(output)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def error_exit(message: str, code: int = 1) -> None:
|
| 156 |
+
"""Print error message and exit."""
|
| 157 |
+
print(f"Error: {message}", file=sys.stderr)
|
| 158 |
+
sys.exit(code)
|
trackio/commit_scheduler.py
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Originally copied from https://github.com/huggingface/huggingface_hub/blob/d0a948fc2a32ed6e557042a95ef3e4af97ec4a7c/src/huggingface_hub/_commit_scheduler.py
|
| 2 |
+
|
| 3 |
+
import atexit
|
| 4 |
+
import logging
|
| 5 |
+
import time
|
| 6 |
+
from concurrent.futures import Future
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from threading import Lock, Thread
|
| 10 |
+
from typing import Callable, Dict, List, Union
|
| 11 |
+
|
| 12 |
+
from huggingface_hub.hf_api import (
|
| 13 |
+
DEFAULT_IGNORE_PATTERNS,
|
| 14 |
+
CommitInfo,
|
| 15 |
+
CommitOperationAdd,
|
| 16 |
+
HfApi,
|
| 17 |
+
)
|
| 18 |
+
from huggingface_hub.utils import filter_repo_objects
|
| 19 |
+
|
| 20 |
+
logger = logging.getLogger(__name__)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass(frozen=True)
|
| 24 |
+
class _FileToUpload:
|
| 25 |
+
"""Temporary dataclass to store info about files to upload. Not meant to be used directly."""
|
| 26 |
+
|
| 27 |
+
local_path: Path
|
| 28 |
+
path_in_repo: str
|
| 29 |
+
size_limit: int
|
| 30 |
+
last_modified: float
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class CommitScheduler:
|
| 34 |
+
"""
|
| 35 |
+
Scheduler to upload a local folder to the Hub at regular intervals (e.g. push to hub every 5 minutes).
|
| 36 |
+
|
| 37 |
+
The recommended way to use the scheduler is to use it as a context manager. This ensures that the scheduler is
|
| 38 |
+
properly stopped and the last commit is triggered when the script ends. The scheduler can also be stopped manually
|
| 39 |
+
with the `stop` method. Checkout the [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#scheduled-uploads)
|
| 40 |
+
to learn more about how to use it.
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
repo_id (`str`):
|
| 44 |
+
The id of the repo to commit to.
|
| 45 |
+
folder_path (`str` or `Path`):
|
| 46 |
+
Path to the local folder to upload regularly.
|
| 47 |
+
every (`int` or `float`, *optional*):
|
| 48 |
+
The number of minutes between each commit. Defaults to 5 minutes.
|
| 49 |
+
path_in_repo (`str`, *optional*):
|
| 50 |
+
Relative path of the directory in the repo, for example: `"checkpoints/"`. Defaults to the root folder
|
| 51 |
+
of the repository.
|
| 52 |
+
repo_type (`str`, *optional*):
|
| 53 |
+
The type of the repo to commit to. Defaults to `model`.
|
| 54 |
+
revision (`str`, *optional*):
|
| 55 |
+
The revision of the repo to commit to. Defaults to `main`.
|
| 56 |
+
private (`bool`, *optional*):
|
| 57 |
+
Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.
|
| 58 |
+
token (`str`, *optional*):
|
| 59 |
+
The token to use to commit to the repo. Defaults to the token saved on the machine.
|
| 60 |
+
allow_patterns (`List[str]` or `str`, *optional*):
|
| 61 |
+
If provided, only files matching at least one pattern are uploaded.
|
| 62 |
+
ignore_patterns (`List[str]` or `str`, *optional*):
|
| 63 |
+
If provided, files matching any of the patterns are not uploaded.
|
| 64 |
+
squash_history (`bool`, *optional*):
|
| 65 |
+
Whether to squash the history of the repo after each commit. Defaults to `False`. Squashing commits is
|
| 66 |
+
useful to avoid degraded performances on the repo when it grows too large.
|
| 67 |
+
hf_api (`HfApi`, *optional*):
|
| 68 |
+
The [`HfApi`] client to use to commit to the Hub. Can be set with custom settings (user agent, token,...).
|
| 69 |
+
on_before_commit (`Callable[[], None]`, *optional*):
|
| 70 |
+
If specified, a function that will be called before the CommitScheduler lists files to create a commit.
|
| 71 |
+
|
| 72 |
+
Example:
|
| 73 |
+
```py
|
| 74 |
+
>>> from pathlib import Path
|
| 75 |
+
>>> from huggingface_hub import CommitScheduler
|
| 76 |
+
|
| 77 |
+
# Scheduler uploads every 10 minutes
|
| 78 |
+
>>> csv_path = Path("watched_folder/data.csv")
|
| 79 |
+
>>> CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path=csv_path.parent, every=10)
|
| 80 |
+
|
| 81 |
+
>>> with csv_path.open("a") as f:
|
| 82 |
+
... f.write("first line")
|
| 83 |
+
|
| 84 |
+
# Some time later (...)
|
| 85 |
+
>>> with csv_path.open("a") as f:
|
| 86 |
+
... f.write("second line")
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
Example using a context manager:
|
| 90 |
+
```py
|
| 91 |
+
>>> from pathlib import Path
|
| 92 |
+
>>> from huggingface_hub import CommitScheduler
|
| 93 |
+
|
| 94 |
+
>>> with CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path="watched_folder", every=10) as scheduler:
|
| 95 |
+
... csv_path = Path("watched_folder/data.csv")
|
| 96 |
+
... with csv_path.open("a") as f:
|
| 97 |
+
... f.write("first line")
|
| 98 |
+
... (...)
|
| 99 |
+
... with csv_path.open("a") as f:
|
| 100 |
+
... f.write("second line")
|
| 101 |
+
|
| 102 |
+
# Scheduler is now stopped and last commit have been triggered
|
| 103 |
+
```
|
| 104 |
+
"""
|
| 105 |
+
|
| 106 |
+
def __init__(
|
| 107 |
+
self,
|
| 108 |
+
*,
|
| 109 |
+
repo_id: str,
|
| 110 |
+
folder_path: Union[str, Path],
|
| 111 |
+
every: Union[int, float] = 5,
|
| 112 |
+
path_in_repo: str | None = None,
|
| 113 |
+
repo_type: str | None = None,
|
| 114 |
+
revision: str | None = None,
|
| 115 |
+
private: bool | None = None,
|
| 116 |
+
token: str | None = None,
|
| 117 |
+
allow_patterns: list[str] | str | None = None,
|
| 118 |
+
ignore_patterns: list[str] | str | None = None,
|
| 119 |
+
squash_history: bool = False,
|
| 120 |
+
hf_api: HfApi | None = None,
|
| 121 |
+
on_before_commit: Callable[[], None] | None = None,
|
| 122 |
+
) -> None:
|
| 123 |
+
self.api = hf_api or HfApi(token=token)
|
| 124 |
+
self.on_before_commit = on_before_commit
|
| 125 |
+
|
| 126 |
+
# Folder
|
| 127 |
+
self.folder_path = Path(folder_path).expanduser().resolve()
|
| 128 |
+
self.path_in_repo = path_in_repo or ""
|
| 129 |
+
self.allow_patterns = allow_patterns
|
| 130 |
+
|
| 131 |
+
if ignore_patterns is None:
|
| 132 |
+
ignore_patterns = []
|
| 133 |
+
elif isinstance(ignore_patterns, str):
|
| 134 |
+
ignore_patterns = [ignore_patterns]
|
| 135 |
+
self.ignore_patterns = ignore_patterns + DEFAULT_IGNORE_PATTERNS
|
| 136 |
+
|
| 137 |
+
if self.folder_path.is_file():
|
| 138 |
+
raise ValueError(
|
| 139 |
+
f"'folder_path' must be a directory, not a file: '{self.folder_path}'."
|
| 140 |
+
)
|
| 141 |
+
self.folder_path.mkdir(parents=True, exist_ok=True)
|
| 142 |
+
|
| 143 |
+
# Repository
|
| 144 |
+
repo_url = self.api.create_repo(
|
| 145 |
+
repo_id=repo_id, private=private, repo_type=repo_type, exist_ok=True
|
| 146 |
+
)
|
| 147 |
+
self.repo_id = repo_url.repo_id
|
| 148 |
+
self.repo_type = repo_type
|
| 149 |
+
self.revision = revision
|
| 150 |
+
self.token = token
|
| 151 |
+
|
| 152 |
+
self.last_uploaded: Dict[Path, float] = {}
|
| 153 |
+
self.last_push_time: float | None = None
|
| 154 |
+
|
| 155 |
+
if not every > 0:
|
| 156 |
+
raise ValueError(f"'every' must be a positive integer, not '{every}'.")
|
| 157 |
+
self.lock = Lock()
|
| 158 |
+
self.every = every
|
| 159 |
+
self.squash_history = squash_history
|
| 160 |
+
|
| 161 |
+
logger.info(
|
| 162 |
+
f"Scheduled job to push '{self.folder_path}' to '{self.repo_id}' every {self.every} minutes."
|
| 163 |
+
)
|
| 164 |
+
self._scheduler_thread = Thread(target=self._run_scheduler, daemon=True)
|
| 165 |
+
self._scheduler_thread.start()
|
| 166 |
+
atexit.register(self._push_to_hub)
|
| 167 |
+
|
| 168 |
+
self.__stopped = False
|
| 169 |
+
|
| 170 |
+
def stop(self) -> None:
|
| 171 |
+
"""Stop the scheduler.
|
| 172 |
+
|
| 173 |
+
A stopped scheduler cannot be restarted. Mostly for tests purposes.
|
| 174 |
+
"""
|
| 175 |
+
self.__stopped = True
|
| 176 |
+
|
| 177 |
+
def __enter__(self) -> "CommitScheduler":
|
| 178 |
+
return self
|
| 179 |
+
|
| 180 |
+
def __exit__(self, exc_type, exc_value, traceback) -> None:
|
| 181 |
+
# Upload last changes before exiting
|
| 182 |
+
self.trigger().result()
|
| 183 |
+
self.stop()
|
| 184 |
+
return
|
| 185 |
+
|
| 186 |
+
def _run_scheduler(self) -> None:
|
| 187 |
+
"""Dumb thread waiting between each scheduled push to Hub."""
|
| 188 |
+
while True:
|
| 189 |
+
self.last_future = self.trigger()
|
| 190 |
+
time.sleep(self.every * 60)
|
| 191 |
+
if self.__stopped:
|
| 192 |
+
break
|
| 193 |
+
|
| 194 |
+
def trigger(self) -> Future:
|
| 195 |
+
"""Trigger a `push_to_hub` and return a future.
|
| 196 |
+
|
| 197 |
+
This method is automatically called every `every` minutes. You can also call it manually to trigger a commit
|
| 198 |
+
immediately, without waiting for the next scheduled commit.
|
| 199 |
+
"""
|
| 200 |
+
return self.api.run_as_future(self._push_to_hub)
|
| 201 |
+
|
| 202 |
+
def _push_to_hub(self) -> CommitInfo | None:
|
| 203 |
+
if self.__stopped: # If stopped, already scheduled commits are ignored
|
| 204 |
+
return None
|
| 205 |
+
|
| 206 |
+
logger.info("(Background) scheduled commit triggered.")
|
| 207 |
+
try:
|
| 208 |
+
value = self.push_to_hub()
|
| 209 |
+
if self.squash_history:
|
| 210 |
+
logger.info("(Background) squashing repo history.")
|
| 211 |
+
self.api.super_squash_history(
|
| 212 |
+
repo_id=self.repo_id, repo_type=self.repo_type, branch=self.revision
|
| 213 |
+
)
|
| 214 |
+
return value
|
| 215 |
+
except Exception as e:
|
| 216 |
+
logger.error(
|
| 217 |
+
f"Error while pushing to Hub: {e}"
|
| 218 |
+
) # Depending on the setup, error might be silenced
|
| 219 |
+
raise
|
| 220 |
+
|
| 221 |
+
def push_to_hub(self) -> CommitInfo | None:
|
| 222 |
+
"""
|
| 223 |
+
Push folder to the Hub and return the commit info.
|
| 224 |
+
|
| 225 |
+
<Tip warning={true}>
|
| 226 |
+
|
| 227 |
+
This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
|
| 228 |
+
queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
|
| 229 |
+
issues.
|
| 230 |
+
|
| 231 |
+
</Tip>
|
| 232 |
+
|
| 233 |
+
The default behavior of `push_to_hub` is to assume an append-only folder. It lists all files in the folder and
|
| 234 |
+
uploads only changed files. If no changes are found, the method returns without committing anything. If you want
|
| 235 |
+
to change this behavior, you can inherit from [`CommitScheduler`] and override this method. This can be useful
|
| 236 |
+
for example to compress data together in a single file before committing. For more details and examples, check
|
| 237 |
+
out our [integration guide](https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#scheduled-uploads).
|
| 238 |
+
"""
|
| 239 |
+
# Check files to upload (with lock)
|
| 240 |
+
with self.lock:
|
| 241 |
+
if self.on_before_commit is not None:
|
| 242 |
+
self.on_before_commit()
|
| 243 |
+
|
| 244 |
+
logger.debug("Listing files to upload for scheduled commit.")
|
| 245 |
+
|
| 246 |
+
# List files from folder (taken from `_prepare_upload_folder_additions`)
|
| 247 |
+
relpath_to_abspath = {
|
| 248 |
+
path.relative_to(self.folder_path).as_posix(): path
|
| 249 |
+
for path in sorted(
|
| 250 |
+
self.folder_path.glob("**/*")
|
| 251 |
+
) # sorted to be deterministic
|
| 252 |
+
if path.is_file()
|
| 253 |
+
}
|
| 254 |
+
prefix = f"{self.path_in_repo.strip('/')}/" if self.path_in_repo else ""
|
| 255 |
+
|
| 256 |
+
# Filter with pattern + filter out unchanged files + retrieve current file size
|
| 257 |
+
files_to_upload: List[_FileToUpload] = []
|
| 258 |
+
for relpath in filter_repo_objects(
|
| 259 |
+
relpath_to_abspath.keys(),
|
| 260 |
+
allow_patterns=self.allow_patterns,
|
| 261 |
+
ignore_patterns=self.ignore_patterns,
|
| 262 |
+
):
|
| 263 |
+
local_path = relpath_to_abspath[relpath]
|
| 264 |
+
stat = local_path.stat()
|
| 265 |
+
if (
|
| 266 |
+
self.last_uploaded.get(local_path) is None
|
| 267 |
+
or self.last_uploaded[local_path] != stat.st_mtime
|
| 268 |
+
):
|
| 269 |
+
files_to_upload.append(
|
| 270 |
+
_FileToUpload(
|
| 271 |
+
local_path=local_path,
|
| 272 |
+
path_in_repo=prefix + relpath,
|
| 273 |
+
size_limit=stat.st_size,
|
| 274 |
+
last_modified=stat.st_mtime,
|
| 275 |
+
)
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
# Return if nothing to upload
|
| 279 |
+
if len(files_to_upload) == 0:
|
| 280 |
+
logger.debug("Dropping schedule commit: no changed file to upload.")
|
| 281 |
+
return None
|
| 282 |
+
|
| 283 |
+
# Convert `_FileToUpload` as `CommitOperationAdd` (=> compute file shas + limit to file size)
|
| 284 |
+
logger.debug("Removing unchanged files since previous scheduled commit.")
|
| 285 |
+
add_operations = [
|
| 286 |
+
CommitOperationAdd(
|
| 287 |
+
# TODO: Cap the file to its current size, even if the user append data to it while a scheduled commit is happening
|
| 288 |
+
# (requires an upstream fix for XET-535: `hf_xet` should support `BinaryIO` for upload)
|
| 289 |
+
path_or_fileobj=file_to_upload.local_path,
|
| 290 |
+
path_in_repo=file_to_upload.path_in_repo,
|
| 291 |
+
)
|
| 292 |
+
for file_to_upload in files_to_upload
|
| 293 |
+
]
|
| 294 |
+
|
| 295 |
+
# Upload files (append mode expected - no need for lock)
|
| 296 |
+
logger.debug("Uploading files for scheduled commit.")
|
| 297 |
+
commit_info = self.api.create_commit(
|
| 298 |
+
repo_id=self.repo_id,
|
| 299 |
+
repo_type=self.repo_type,
|
| 300 |
+
operations=add_operations,
|
| 301 |
+
commit_message="Scheduled Commit",
|
| 302 |
+
revision=self.revision,
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
for file in files_to_upload:
|
| 306 |
+
self.last_uploaded[file.local_path] = file.last_modified
|
| 307 |
+
|
| 308 |
+
self.last_push_time = time.time()
|
| 309 |
+
|
| 310 |
+
return commit_info
|
trackio/context_vars.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import contextvars
|
| 2 |
+
from typing import TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
if TYPE_CHECKING:
|
| 5 |
+
from trackio.run import Run
|
| 6 |
+
|
| 7 |
+
current_run: contextvars.ContextVar["Run | None"] = contextvars.ContextVar(
|
| 8 |
+
"current_run", default=None
|
| 9 |
+
)
|
| 10 |
+
current_project: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
| 11 |
+
"current_project", default=None
|
| 12 |
+
)
|
| 13 |
+
current_server: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
| 14 |
+
"current_server", default=None
|
| 15 |
+
)
|
| 16 |
+
current_space_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
| 17 |
+
"current_space_id", default=None
|
| 18 |
+
)
|
trackio/deploy.py
ADDED
|
@@ -0,0 +1,938 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib.metadata
|
| 2 |
+
import io
|
| 3 |
+
import json as json_mod
|
| 4 |
+
import os
|
| 5 |
+
import shutil
|
| 6 |
+
import sys
|
| 7 |
+
import tempfile
|
| 8 |
+
import threading
|
| 9 |
+
import time
|
| 10 |
+
from importlib.resources import files
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
if sys.version_info >= (3, 11):
|
| 14 |
+
import tomllib
|
| 15 |
+
else:
|
| 16 |
+
import tomli as tomllib
|
| 17 |
+
|
| 18 |
+
import httpx
|
| 19 |
+
import huggingface_hub
|
| 20 |
+
from gradio_client import Client, handle_file
|
| 21 |
+
from httpx import ReadTimeout
|
| 22 |
+
from huggingface_hub import Volume
|
| 23 |
+
from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError
|
| 24 |
+
|
| 25 |
+
import trackio
|
| 26 |
+
from trackio.bucket_storage import (
|
| 27 |
+
create_bucket_if_not_exists,
|
| 28 |
+
export_from_bucket_for_static,
|
| 29 |
+
upload_project_to_bucket,
|
| 30 |
+
upload_project_to_bucket_for_static,
|
| 31 |
+
)
|
| 32 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 33 |
+
from trackio.utils import (
|
| 34 |
+
MEDIA_DIR,
|
| 35 |
+
get_or_create_project_hash,
|
| 36 |
+
preprocess_space_and_dataset_ids,
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
SPACE_HOST_URL = "https://{user_name}-{space_name}.hf.space/"
|
| 40 |
+
SPACE_URL = "https://huggingface.co/spaces/{space_id}"
|
| 41 |
+
_BOLD_ORANGE = "\033[1m\033[38;5;208m"
|
| 42 |
+
_RESET = "\033[0m"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def raise_if_space_is_frozen_for_logging(space_id: str) -> None:
|
| 46 |
+
try:
|
| 47 |
+
info = huggingface_hub.HfApi().space_info(space_id)
|
| 48 |
+
except RepositoryNotFoundError:
|
| 49 |
+
return
|
| 50 |
+
if getattr(info, "sdk", None) == "static":
|
| 51 |
+
raise RuntimeError(
|
| 52 |
+
f"Cannot log to Hugging Face Space '{space_id}' because it has been frozen "
|
| 53 |
+
f"(it uses the static SDK: a read-only dashboard with no live Trackio server).\n\n"
|
| 54 |
+
f"Use a different space_id for training, or create a new Gradio Trackio Space. "
|
| 55 |
+
f"Freezing converts a live Gradio Space to static after a run; a frozen Space "
|
| 56 |
+
f'cannot accept new logs. See trackio.sync(..., sdk="static") in the Trackio docs.'
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _readme_linked_hub_yaml(dataset_id: str | None) -> str:
|
| 61 |
+
if dataset_id is not None:
|
| 62 |
+
return f"datasets:\n - {dataset_id}\n"
|
| 63 |
+
return ""
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
_SPACE_APP_PY = "import trackio\ntrackio.show()\n"
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _retry_hf_write(op_name: str, fn, retries: int = 4, initial_delay: float = 1.5):
|
| 70 |
+
delay = initial_delay
|
| 71 |
+
for attempt in range(1, retries + 1):
|
| 72 |
+
try:
|
| 73 |
+
return fn()
|
| 74 |
+
except ReadTimeout:
|
| 75 |
+
if attempt == retries:
|
| 76 |
+
raise
|
| 77 |
+
print(
|
| 78 |
+
f"* {op_name} timed out (attempt {attempt}/{retries}). Retrying in {delay:.1f}s..."
|
| 79 |
+
)
|
| 80 |
+
time.sleep(delay)
|
| 81 |
+
delay = min(delay * 2, 12)
|
| 82 |
+
except HfHubHTTPError as e:
|
| 83 |
+
status = e.response.status_code if e.response is not None else None
|
| 84 |
+
if status is None or status < 500 or attempt == retries:
|
| 85 |
+
raise
|
| 86 |
+
print(
|
| 87 |
+
f"* {op_name} failed with HTTP {status} (attempt {attempt}/{retries}). Retrying in {delay:.1f}s..."
|
| 88 |
+
)
|
| 89 |
+
time.sleep(delay)
|
| 90 |
+
delay = min(delay * 2, 12)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _get_space_volumes(space_id: str) -> list[Volume]:
|
| 94 |
+
"""
|
| 95 |
+
Return mounted volumes for a Space.
|
| 96 |
+
|
| 97 |
+
`HfApi.get_space_runtime()` does not always populate `volumes`, even when the
|
| 98 |
+
mount exists. Fall back to `space_info().runtime.volumes`, which currently
|
| 99 |
+
carries the volume metadata for running Spaces.
|
| 100 |
+
"""
|
| 101 |
+
hf_api = huggingface_hub.HfApi()
|
| 102 |
+
runtime = hf_api.get_space_runtime(space_id)
|
| 103 |
+
if runtime.volumes:
|
| 104 |
+
return list(runtime.volumes)
|
| 105 |
+
|
| 106 |
+
info = hf_api.space_info(space_id)
|
| 107 |
+
if info.runtime and info.runtime.volumes:
|
| 108 |
+
return list(info.runtime.volumes)
|
| 109 |
+
|
| 110 |
+
return []
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _get_source_install_dependencies() -> str:
|
| 114 |
+
"""Get trackio dependencies from pyproject.toml for source installs."""
|
| 115 |
+
trackio_path = files("trackio")
|
| 116 |
+
pyproject_path = Path(trackio_path).parent / "pyproject.toml"
|
| 117 |
+
with open(pyproject_path, "rb") as f:
|
| 118 |
+
pyproject = tomllib.load(f)
|
| 119 |
+
deps = pyproject["project"]["dependencies"]
|
| 120 |
+
spaces_deps = (
|
| 121 |
+
pyproject["project"].get("optional-dependencies", {}).get("spaces", [])
|
| 122 |
+
)
|
| 123 |
+
return "\n".join(deps + spaces_deps)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _is_trackio_installed_from_source() -> bool:
|
| 127 |
+
"""Check if trackio is installed from source/editable install vs PyPI."""
|
| 128 |
+
try:
|
| 129 |
+
trackio_file = trackio.__file__
|
| 130 |
+
if "site-packages" not in trackio_file and "dist-packages" not in trackio_file:
|
| 131 |
+
return True
|
| 132 |
+
|
| 133 |
+
dist = importlib.metadata.distribution("trackio")
|
| 134 |
+
if dist.files:
|
| 135 |
+
files = list(dist.files)
|
| 136 |
+
has_pth = any(".pth" in str(f) for f in files)
|
| 137 |
+
if has_pth:
|
| 138 |
+
return True
|
| 139 |
+
|
| 140 |
+
return False
|
| 141 |
+
except (
|
| 142 |
+
AttributeError,
|
| 143 |
+
importlib.metadata.PackageNotFoundError,
|
| 144 |
+
importlib.metadata.MetadataError,
|
| 145 |
+
ValueError,
|
| 146 |
+
TypeError,
|
| 147 |
+
):
|
| 148 |
+
return True
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def deploy_as_space(
|
| 152 |
+
space_id: str,
|
| 153 |
+
space_storage: huggingface_hub.SpaceStorage | None = None,
|
| 154 |
+
dataset_id: str | None = None,
|
| 155 |
+
bucket_id: str | None = None,
|
| 156 |
+
private: bool | None = None,
|
| 157 |
+
):
|
| 158 |
+
if (
|
| 159 |
+
os.getenv("SYSTEM") == "spaces"
|
| 160 |
+
): # in case a repo with this function is uploaded to spaces
|
| 161 |
+
return
|
| 162 |
+
|
| 163 |
+
if dataset_id is not None and bucket_id is not None:
|
| 164 |
+
raise ValueError(
|
| 165 |
+
"Cannot use bucket volume options together with dataset_id; use one persistence mode."
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
trackio_path = files("trackio")
|
| 169 |
+
|
| 170 |
+
hf_api = huggingface_hub.HfApi()
|
| 171 |
+
|
| 172 |
+
try:
|
| 173 |
+
huggingface_hub.create_repo(
|
| 174 |
+
space_id,
|
| 175 |
+
private=private,
|
| 176 |
+
space_sdk="gradio",
|
| 177 |
+
space_storage=space_storage,
|
| 178 |
+
repo_type="space",
|
| 179 |
+
exist_ok=True,
|
| 180 |
+
)
|
| 181 |
+
except HfHubHTTPError as e:
|
| 182 |
+
if e.response.status_code in [401, 403]: # unauthorized or forbidden
|
| 183 |
+
print("Need 'write' access token to create a Spaces repo.")
|
| 184 |
+
huggingface_hub.login(add_to_git_credential=False)
|
| 185 |
+
huggingface_hub.create_repo(
|
| 186 |
+
space_id,
|
| 187 |
+
private=private,
|
| 188 |
+
space_sdk="gradio",
|
| 189 |
+
space_storage=space_storage,
|
| 190 |
+
repo_type="space",
|
| 191 |
+
exist_ok=True,
|
| 192 |
+
)
|
| 193 |
+
else:
|
| 194 |
+
raise ValueError(f"Failed to create Space: {e}")
|
| 195 |
+
|
| 196 |
+
# We can assume pandas and huggingface-hub are available; requirements.txt pins trackio.
|
| 197 |
+
# Make sure necessary dependencies are installed by creating a requirements.txt.
|
| 198 |
+
is_source_install = _is_trackio_installed_from_source()
|
| 199 |
+
|
| 200 |
+
if bucket_id is not None:
|
| 201 |
+
create_bucket_if_not_exists(bucket_id, private=private)
|
| 202 |
+
|
| 203 |
+
with open(Path(trackio_path, "README.md"), "r") as f:
|
| 204 |
+
readme_content = f.read()
|
| 205 |
+
readme_content = readme_content.replace("{GRADIO_VERSION}", trackio.__version__)
|
| 206 |
+
readme_content = readme_content.replace("{APP_FILE}", "app.py")
|
| 207 |
+
readme_content = readme_content.replace(
|
| 208 |
+
"{LINKED_HUB_METADATA}", _readme_linked_hub_yaml(dataset_id)
|
| 209 |
+
)
|
| 210 |
+
readme_buffer = io.BytesIO(readme_content.encode("utf-8"))
|
| 211 |
+
hf_api.upload_file(
|
| 212 |
+
path_or_fileobj=readme_buffer,
|
| 213 |
+
path_in_repo="README.md",
|
| 214 |
+
repo_id=space_id,
|
| 215 |
+
repo_type="space",
|
| 216 |
+
)
|
| 217 |
+
|
| 218 |
+
if is_source_install:
|
| 219 |
+
requirements_content = _get_source_install_dependencies()
|
| 220 |
+
else:
|
| 221 |
+
requirements_content = f"trackio[spaces]=={trackio.__version__}"
|
| 222 |
+
|
| 223 |
+
requirements_buffer = io.BytesIO(requirements_content.encode("utf-8"))
|
| 224 |
+
hf_api.upload_file(
|
| 225 |
+
path_or_fileobj=requirements_buffer,
|
| 226 |
+
path_in_repo="requirements.txt",
|
| 227 |
+
repo_id=space_id,
|
| 228 |
+
repo_type="space",
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
huggingface_hub.utils.disable_progress_bars()
|
| 232 |
+
|
| 233 |
+
if is_source_install:
|
| 234 |
+
dist_index = (
|
| 235 |
+
Path(trackio.__file__).resolve().parent / "frontend" / "dist" / "index.html"
|
| 236 |
+
)
|
| 237 |
+
if not dist_index.is_file():
|
| 238 |
+
raise ValueError(
|
| 239 |
+
"The Trackio frontend build is missing. From the repository root run "
|
| 240 |
+
"`cd trackio/frontend && npm ci && npm run build`, then deploy again."
|
| 241 |
+
)
|
| 242 |
+
hf_api.upload_folder(
|
| 243 |
+
repo_id=space_id,
|
| 244 |
+
repo_type="space",
|
| 245 |
+
folder_path=trackio_path,
|
| 246 |
+
path_in_repo="trackio",
|
| 247 |
+
ignore_patterns=[
|
| 248 |
+
"README.md",
|
| 249 |
+
"frontend/node_modules/**",
|
| 250 |
+
"frontend/src/**",
|
| 251 |
+
"frontend/.gitignore",
|
| 252 |
+
"frontend/package.json",
|
| 253 |
+
"frontend/package-lock.json",
|
| 254 |
+
"frontend/vite.config.js",
|
| 255 |
+
"frontend/svelte.config.js",
|
| 256 |
+
"**/__pycache__/**",
|
| 257 |
+
"*.pyc",
|
| 258 |
+
],
|
| 259 |
+
)
|
| 260 |
+
|
| 261 |
+
app_file_content = _SPACE_APP_PY
|
| 262 |
+
app_file_buffer = io.BytesIO(app_file_content.encode("utf-8"))
|
| 263 |
+
hf_api.upload_file(
|
| 264 |
+
path_or_fileobj=app_file_buffer,
|
| 265 |
+
path_in_repo="app.py",
|
| 266 |
+
repo_id=space_id,
|
| 267 |
+
repo_type="space",
|
| 268 |
+
)
|
| 269 |
+
|
| 270 |
+
if hf_token := huggingface_hub.utils.get_token():
|
| 271 |
+
huggingface_hub.add_space_secret(space_id, "HF_TOKEN", hf_token)
|
| 272 |
+
if bucket_id is not None:
|
| 273 |
+
existing = _get_space_volumes(space_id)
|
| 274 |
+
already_mounted = any(
|
| 275 |
+
v.type == "bucket" and v.source == bucket_id and v.mount_path == "/data"
|
| 276 |
+
for v in existing
|
| 277 |
+
)
|
| 278 |
+
if not already_mounted:
|
| 279 |
+
non_bucket = [
|
| 280 |
+
v
|
| 281 |
+
for v in existing
|
| 282 |
+
if not (v.type == "bucket" and v.source == bucket_id)
|
| 283 |
+
]
|
| 284 |
+
hf_api.set_space_volumes(
|
| 285 |
+
space_id,
|
| 286 |
+
non_bucket
|
| 287 |
+
+ [Volume(type="bucket", source=bucket_id, mount_path="/data")],
|
| 288 |
+
)
|
| 289 |
+
print(f"* Attached bucket {bucket_id} at '/data'")
|
| 290 |
+
huggingface_hub.add_space_variable(space_id, "TRACKIO_DIR", "/data/trackio")
|
| 291 |
+
elif dataset_id is not None:
|
| 292 |
+
huggingface_hub.add_space_variable(space_id, "TRACKIO_DATASET_ID", dataset_id)
|
| 293 |
+
if logo_light_url := os.environ.get("TRACKIO_LOGO_LIGHT_URL"):
|
| 294 |
+
huggingface_hub.add_space_variable(
|
| 295 |
+
space_id, "TRACKIO_LOGO_LIGHT_URL", logo_light_url
|
| 296 |
+
)
|
| 297 |
+
if logo_dark_url := os.environ.get("TRACKIO_LOGO_DARK_URL"):
|
| 298 |
+
huggingface_hub.add_space_variable(
|
| 299 |
+
space_id, "TRACKIO_LOGO_DARK_URL", logo_dark_url
|
| 300 |
+
)
|
| 301 |
+
if plot_order := os.environ.get("TRACKIO_PLOT_ORDER"):
|
| 302 |
+
huggingface_hub.add_space_variable(space_id, "TRACKIO_PLOT_ORDER", plot_order)
|
| 303 |
+
if theme := os.environ.get("TRACKIO_THEME"):
|
| 304 |
+
huggingface_hub.add_space_variable(space_id, "TRACKIO_THEME", theme)
|
| 305 |
+
huggingface_hub.add_space_variable(space_id, "GRADIO_MCP_SERVER", "True")
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def create_space_if_not_exists(
|
| 309 |
+
space_id: str,
|
| 310 |
+
space_storage: huggingface_hub.SpaceStorage | None = None,
|
| 311 |
+
dataset_id: str | None = None,
|
| 312 |
+
bucket_id: str | None = None,
|
| 313 |
+
private: bool | None = None,
|
| 314 |
+
) -> None:
|
| 315 |
+
"""
|
| 316 |
+
Creates a new Hugging Face Space if it does not exist.
|
| 317 |
+
|
| 318 |
+
Args:
|
| 319 |
+
space_id (`str`):
|
| 320 |
+
The ID of the Space to create.
|
| 321 |
+
space_storage ([`~huggingface_hub.SpaceStorage`], *optional*):
|
| 322 |
+
Choice of persistent storage tier for the Space.
|
| 323 |
+
dataset_id (`str`, *optional*):
|
| 324 |
+
Deprecated. Use `bucket_id` instead.
|
| 325 |
+
bucket_id (`str`, *optional*):
|
| 326 |
+
Full Hub bucket id (`namespace/name`) to attach via the Hub volumes API (platform mount).
|
| 327 |
+
Sets `TRACKIO_DIR` to the mount path.
|
| 328 |
+
private (`bool`, *optional*):
|
| 329 |
+
Whether to make the Space private. If `None` (default), the repo will be
|
| 330 |
+
public unless the organization's default is private. This value is ignored
|
| 331 |
+
if the repo already exists.
|
| 332 |
+
"""
|
| 333 |
+
if "/" not in space_id:
|
| 334 |
+
raise ValueError(
|
| 335 |
+
f"Invalid space ID: {space_id}. Must be in the format: username/reponame or orgname/reponame."
|
| 336 |
+
)
|
| 337 |
+
if dataset_id is not None and "/" not in dataset_id:
|
| 338 |
+
raise ValueError(
|
| 339 |
+
f"Invalid dataset ID: {dataset_id}. Must be in the format: username/datasetname or orgname/datasetname."
|
| 340 |
+
)
|
| 341 |
+
if bucket_id is not None and "/" not in bucket_id:
|
| 342 |
+
raise ValueError(
|
| 343 |
+
f"Invalid bucket ID: {bucket_id}. Must be in the format: username/bucketname or orgname/bucketname."
|
| 344 |
+
)
|
| 345 |
+
try:
|
| 346 |
+
huggingface_hub.repo_info(space_id, repo_type="space")
|
| 347 |
+
print(
|
| 348 |
+
f"* Found existing space: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
|
| 349 |
+
)
|
| 350 |
+
return
|
| 351 |
+
except RepositoryNotFoundError:
|
| 352 |
+
pass
|
| 353 |
+
except HfHubHTTPError as e:
|
| 354 |
+
if e.response.status_code in [401, 403]: # unauthorized or forbidden
|
| 355 |
+
print("Need 'write' access token to create a Spaces repo.")
|
| 356 |
+
huggingface_hub.login(add_to_git_credential=False)
|
| 357 |
+
else:
|
| 358 |
+
raise ValueError(f"Failed to create Space: {e}")
|
| 359 |
+
|
| 360 |
+
print(
|
| 361 |
+
f"* Creating new space: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
|
| 362 |
+
)
|
| 363 |
+
deploy_as_space(
|
| 364 |
+
space_id,
|
| 365 |
+
space_storage,
|
| 366 |
+
dataset_id,
|
| 367 |
+
bucket_id,
|
| 368 |
+
private,
|
| 369 |
+
)
|
| 370 |
+
print("* Waiting for Space to be ready...")
|
| 371 |
+
_wait_until_space_running(space_id)
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def _wait_until_space_running(space_id: str, timeout: int = 300) -> None:
|
| 375 |
+
hf_api = huggingface_hub.HfApi()
|
| 376 |
+
start = time.time()
|
| 377 |
+
delay = 2
|
| 378 |
+
request_timeout = 45.0
|
| 379 |
+
failure_stages = frozenset(
|
| 380 |
+
("NO_APP_FILE", "CONFIG_ERROR", "BUILD_ERROR", "RUNTIME_ERROR")
|
| 381 |
+
)
|
| 382 |
+
while time.time() - start < timeout:
|
| 383 |
+
try:
|
| 384 |
+
info = hf_api.space_info(space_id, timeout=request_timeout)
|
| 385 |
+
if info.runtime:
|
| 386 |
+
stage = str(info.runtime.stage)
|
| 387 |
+
if stage in failure_stages:
|
| 388 |
+
raise RuntimeError(
|
| 389 |
+
f"Space {space_id} entered terminal stage {stage}. "
|
| 390 |
+
"Fix README.md or app files; see build logs on the Hub."
|
| 391 |
+
)
|
| 392 |
+
if stage == "RUNNING":
|
| 393 |
+
return
|
| 394 |
+
except RuntimeError:
|
| 395 |
+
raise
|
| 396 |
+
except (huggingface_hub.utils.HfHubHTTPError, httpx.RequestError):
|
| 397 |
+
pass
|
| 398 |
+
time.sleep(delay)
|
| 399 |
+
delay = min(delay * 1.5, 15)
|
| 400 |
+
raise TimeoutError(
|
| 401 |
+
f"Space {space_id} did not reach RUNNING within {timeout}s. "
|
| 402 |
+
"Check status and build logs on the Hub."
|
| 403 |
+
)
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def wait_until_space_exists(
|
| 407 |
+
space_id: str,
|
| 408 |
+
) -> None:
|
| 409 |
+
"""
|
| 410 |
+
Blocks the current thread until the Space exists.
|
| 411 |
+
|
| 412 |
+
Args:
|
| 413 |
+
space_id (`str`):
|
| 414 |
+
The ID of the Space to wait for.
|
| 415 |
+
|
| 416 |
+
Raises:
|
| 417 |
+
`TimeoutError`: If waiting for the Space takes longer than expected.
|
| 418 |
+
"""
|
| 419 |
+
hf_api = huggingface_hub.HfApi()
|
| 420 |
+
delay = 1
|
| 421 |
+
for _ in range(30):
|
| 422 |
+
try:
|
| 423 |
+
hf_api.space_info(space_id)
|
| 424 |
+
return
|
| 425 |
+
except (huggingface_hub.utils.HfHubHTTPError, httpx.RequestError):
|
| 426 |
+
time.sleep(delay)
|
| 427 |
+
delay = min(delay * 2, 60)
|
| 428 |
+
raise TimeoutError("Waiting for space to exist took longer than expected")
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def upload_db_to_space(project: str, space_id: str, force: bool = False) -> None:
|
| 432 |
+
"""
|
| 433 |
+
Uploads the database of a local Trackio project to a Hugging Face Space.
|
| 434 |
+
|
| 435 |
+
This uses the Gradio Client to upload since we do not want to trigger a new build of
|
| 436 |
+
the Space, which would happen if we used `huggingface_hub.upload_file`.
|
| 437 |
+
|
| 438 |
+
Args:
|
| 439 |
+
project (`str`):
|
| 440 |
+
The name of the project to upload.
|
| 441 |
+
space_id (`str`):
|
| 442 |
+
The ID of the Space to upload to.
|
| 443 |
+
force (`bool`, *optional*, defaults to `False`):
|
| 444 |
+
If `True`, overwrites the existing database without prompting. If `False`,
|
| 445 |
+
prompts for confirmation.
|
| 446 |
+
"""
|
| 447 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 448 |
+
client = Client(space_id, verbose=False, httpx_kwargs={"timeout": 90})
|
| 449 |
+
|
| 450 |
+
if not force:
|
| 451 |
+
try:
|
| 452 |
+
existing_projects = client.predict(api_name="/get_all_projects")
|
| 453 |
+
if project in existing_projects:
|
| 454 |
+
response = input(
|
| 455 |
+
f"Database for project '{project}' already exists on Space '{space_id}'. "
|
| 456 |
+
f"Overwrite it? (y/N): "
|
| 457 |
+
)
|
| 458 |
+
if response.lower() not in ["y", "yes"]:
|
| 459 |
+
print("* Upload cancelled.")
|
| 460 |
+
return
|
| 461 |
+
except Exception as e:
|
| 462 |
+
print(f"* Warning: Could not check if project exists on Space: {e}")
|
| 463 |
+
print("* Proceeding with upload...")
|
| 464 |
+
|
| 465 |
+
client.predict(
|
| 466 |
+
api_name="/upload_db_to_space",
|
| 467 |
+
project=project,
|
| 468 |
+
uploaded_db=handle_file(db_path),
|
| 469 |
+
hf_token=huggingface_hub.utils.get_token(),
|
| 470 |
+
)
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
SYNC_BATCH_SIZE = 500
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
def sync_incremental(
|
| 477 |
+
project: str,
|
| 478 |
+
space_id: str,
|
| 479 |
+
private: bool | None = None,
|
| 480 |
+
pending_only: bool = False,
|
| 481 |
+
) -> None:
|
| 482 |
+
"""
|
| 483 |
+
Syncs a local Trackio project to a Space via the bulk_log API endpoints
|
| 484 |
+
instead of uploading the entire DB file. Supports incremental sync.
|
| 485 |
+
|
| 486 |
+
Args:
|
| 487 |
+
project: The name of the project to sync.
|
| 488 |
+
space_id: The HF Space ID to sync to.
|
| 489 |
+
private: Whether to make the Space private if creating.
|
| 490 |
+
pending_only: If True, only sync rows tagged with space_id (pending data).
|
| 491 |
+
"""
|
| 492 |
+
print(
|
| 493 |
+
f"* Syncing project '{project}' to: {SPACE_URL.format(space_id=space_id)} (please wait...)"
|
| 494 |
+
)
|
| 495 |
+
create_space_if_not_exists(space_id, private=private)
|
| 496 |
+
wait_until_space_exists(space_id)
|
| 497 |
+
|
| 498 |
+
client = Client(space_id, verbose=False, httpx_kwargs={"timeout": 90})
|
| 499 |
+
hf_token = huggingface_hub.utils.get_token()
|
| 500 |
+
|
| 501 |
+
if pending_only:
|
| 502 |
+
pending_logs = SQLiteStorage.get_pending_logs(project)
|
| 503 |
+
if pending_logs:
|
| 504 |
+
logs = pending_logs["logs"]
|
| 505 |
+
for i in range(0, len(logs), SYNC_BATCH_SIZE):
|
| 506 |
+
batch = logs[i : i + SYNC_BATCH_SIZE]
|
| 507 |
+
print(
|
| 508 |
+
f" Syncing metrics: {min(i + SYNC_BATCH_SIZE, len(logs))}/{len(logs)}..."
|
| 509 |
+
)
|
| 510 |
+
client.predict(api_name="/bulk_log", logs=batch, hf_token=hf_token)
|
| 511 |
+
SQLiteStorage.clear_pending_logs(project, pending_logs["ids"])
|
| 512 |
+
|
| 513 |
+
pending_sys = SQLiteStorage.get_pending_system_logs(project)
|
| 514 |
+
if pending_sys:
|
| 515 |
+
logs = pending_sys["logs"]
|
| 516 |
+
for i in range(0, len(logs), SYNC_BATCH_SIZE):
|
| 517 |
+
batch = logs[i : i + SYNC_BATCH_SIZE]
|
| 518 |
+
print(
|
| 519 |
+
f" Syncing system metrics: {min(i + SYNC_BATCH_SIZE, len(logs))}/{len(logs)}..."
|
| 520 |
+
)
|
| 521 |
+
client.predict(
|
| 522 |
+
api_name="/bulk_log_system", logs=batch, hf_token=hf_token
|
| 523 |
+
)
|
| 524 |
+
SQLiteStorage.clear_pending_system_logs(project, pending_sys["ids"])
|
| 525 |
+
|
| 526 |
+
pending_uploads = SQLiteStorage.get_pending_uploads(project)
|
| 527 |
+
if pending_uploads:
|
| 528 |
+
upload_entries = []
|
| 529 |
+
for u in pending_uploads["uploads"]:
|
| 530 |
+
fp = u["file_path"]
|
| 531 |
+
if os.path.exists(fp):
|
| 532 |
+
upload_entries.append(
|
| 533 |
+
{
|
| 534 |
+
"project": u["project"],
|
| 535 |
+
"run": u["run"],
|
| 536 |
+
"step": u["step"],
|
| 537 |
+
"relative_path": u["relative_path"],
|
| 538 |
+
"uploaded_file": handle_file(fp),
|
| 539 |
+
}
|
| 540 |
+
)
|
| 541 |
+
if upload_entries:
|
| 542 |
+
print(f" Syncing {len(upload_entries)} media files...")
|
| 543 |
+
client.predict(
|
| 544 |
+
api_name="/bulk_upload_media",
|
| 545 |
+
uploads=upload_entries,
|
| 546 |
+
hf_token=hf_token,
|
| 547 |
+
)
|
| 548 |
+
SQLiteStorage.clear_pending_uploads(project, pending_uploads["ids"])
|
| 549 |
+
else:
|
| 550 |
+
all_logs = SQLiteStorage.get_all_logs_for_sync(project)
|
| 551 |
+
if all_logs:
|
| 552 |
+
for i in range(0, len(all_logs), SYNC_BATCH_SIZE):
|
| 553 |
+
batch = all_logs[i : i + SYNC_BATCH_SIZE]
|
| 554 |
+
print(
|
| 555 |
+
f" Syncing metrics: {min(i + SYNC_BATCH_SIZE, len(all_logs))}/{len(all_logs)}..."
|
| 556 |
+
)
|
| 557 |
+
client.predict(api_name="/bulk_log", logs=batch, hf_token=hf_token)
|
| 558 |
+
|
| 559 |
+
all_sys_logs = SQLiteStorage.get_all_system_logs_for_sync(project)
|
| 560 |
+
if all_sys_logs:
|
| 561 |
+
for i in range(0, len(all_sys_logs), SYNC_BATCH_SIZE):
|
| 562 |
+
batch = all_sys_logs[i : i + SYNC_BATCH_SIZE]
|
| 563 |
+
print(
|
| 564 |
+
f" Syncing system metrics: {min(i + SYNC_BATCH_SIZE, len(all_sys_logs))}/{len(all_sys_logs)}..."
|
| 565 |
+
)
|
| 566 |
+
client.predict(
|
| 567 |
+
api_name="/bulk_log_system", logs=batch, hf_token=hf_token
|
| 568 |
+
)
|
| 569 |
+
|
| 570 |
+
SQLiteStorage.set_project_metadata(project, "space_id", space_id)
|
| 571 |
+
print(
|
| 572 |
+
f"* Synced successfully to space: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
|
| 573 |
+
)
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
def upload_dataset_for_static(
|
| 577 |
+
project: str,
|
| 578 |
+
dataset_id: str,
|
| 579 |
+
private: bool | None = None,
|
| 580 |
+
) -> None:
|
| 581 |
+
hf_api = huggingface_hub.HfApi()
|
| 582 |
+
|
| 583 |
+
try:
|
| 584 |
+
huggingface_hub.create_repo(
|
| 585 |
+
dataset_id,
|
| 586 |
+
private=private,
|
| 587 |
+
repo_type="dataset",
|
| 588 |
+
exist_ok=True,
|
| 589 |
+
)
|
| 590 |
+
except HfHubHTTPError as e:
|
| 591 |
+
if e.response.status_code in [401, 403]:
|
| 592 |
+
print("Need 'write' access token to create a Dataset repo.")
|
| 593 |
+
huggingface_hub.login(add_to_git_credential=False)
|
| 594 |
+
huggingface_hub.create_repo(
|
| 595 |
+
dataset_id,
|
| 596 |
+
private=private,
|
| 597 |
+
repo_type="dataset",
|
| 598 |
+
exist_ok=True,
|
| 599 |
+
)
|
| 600 |
+
else:
|
| 601 |
+
raise ValueError(f"Failed to create Dataset: {e}")
|
| 602 |
+
|
| 603 |
+
with tempfile.TemporaryDirectory() as tmp_dir:
|
| 604 |
+
output_dir = Path(tmp_dir)
|
| 605 |
+
SQLiteStorage.export_for_static_space(project, output_dir)
|
| 606 |
+
|
| 607 |
+
media_dir = MEDIA_DIR / project
|
| 608 |
+
if media_dir.exists():
|
| 609 |
+
dest = output_dir / "media"
|
| 610 |
+
shutil.copytree(media_dir, dest)
|
| 611 |
+
|
| 612 |
+
_retry_hf_write(
|
| 613 |
+
"Dataset upload",
|
| 614 |
+
lambda: hf_api.upload_folder(
|
| 615 |
+
repo_id=dataset_id,
|
| 616 |
+
repo_type="dataset",
|
| 617 |
+
folder_path=str(output_dir),
|
| 618 |
+
),
|
| 619 |
+
)
|
| 620 |
+
|
| 621 |
+
print(f"* Dataset uploaded: https://huggingface.co/datasets/{dataset_id}")
|
| 622 |
+
|
| 623 |
+
|
| 624 |
+
def deploy_as_static_space(
|
| 625 |
+
space_id: str,
|
| 626 |
+
dataset_id: str | None,
|
| 627 |
+
project: str,
|
| 628 |
+
bucket_id: str | None = None,
|
| 629 |
+
private: bool | None = None,
|
| 630 |
+
hf_token: str | None = None,
|
| 631 |
+
) -> None:
|
| 632 |
+
if os.getenv("SYSTEM") == "spaces":
|
| 633 |
+
return
|
| 634 |
+
|
| 635 |
+
hf_api = huggingface_hub.HfApi()
|
| 636 |
+
|
| 637 |
+
try:
|
| 638 |
+
huggingface_hub.create_repo(
|
| 639 |
+
space_id,
|
| 640 |
+
private=private,
|
| 641 |
+
space_sdk="static",
|
| 642 |
+
repo_type="space",
|
| 643 |
+
exist_ok=True,
|
| 644 |
+
)
|
| 645 |
+
except HfHubHTTPError as e:
|
| 646 |
+
if e.response.status_code in [401, 403]:
|
| 647 |
+
print("Need 'write' access token to create a Spaces repo.")
|
| 648 |
+
huggingface_hub.login(add_to_git_credential=False)
|
| 649 |
+
huggingface_hub.create_repo(
|
| 650 |
+
space_id,
|
| 651 |
+
private=private,
|
| 652 |
+
space_sdk="static",
|
| 653 |
+
repo_type="space",
|
| 654 |
+
exist_ok=True,
|
| 655 |
+
)
|
| 656 |
+
else:
|
| 657 |
+
raise ValueError(f"Failed to create Space: {e}")
|
| 658 |
+
|
| 659 |
+
linked = _readme_linked_hub_yaml(dataset_id)
|
| 660 |
+
readme_content = (
|
| 661 |
+
f"---\nsdk: static\npinned: false\ntags:\n - trackio\n{linked}---\n"
|
| 662 |
+
)
|
| 663 |
+
_retry_hf_write(
|
| 664 |
+
"Static Space README upload",
|
| 665 |
+
lambda: hf_api.upload_file(
|
| 666 |
+
path_or_fileobj=io.BytesIO(readme_content.encode("utf-8")),
|
| 667 |
+
path_in_repo="README.md",
|
| 668 |
+
repo_id=space_id,
|
| 669 |
+
repo_type="space",
|
| 670 |
+
),
|
| 671 |
+
)
|
| 672 |
+
|
| 673 |
+
trackio_path = files("trackio")
|
| 674 |
+
dist_dir = Path(trackio_path).parent / "trackio" / "frontend" / "dist"
|
| 675 |
+
if not dist_dir.is_dir():
|
| 676 |
+
dist_dir = Path(trackio.__file__).resolve().parent / "frontend" / "dist"
|
| 677 |
+
if not dist_dir.is_dir():
|
| 678 |
+
raise ValueError(
|
| 679 |
+
"The Trackio frontend build is missing. From the repository root run "
|
| 680 |
+
"`cd trackio/frontend && npm ci && npm run build`, then deploy again."
|
| 681 |
+
)
|
| 682 |
+
|
| 683 |
+
_retry_hf_write(
|
| 684 |
+
"Static Space frontend upload",
|
| 685 |
+
lambda: hf_api.upload_folder(
|
| 686 |
+
repo_id=space_id,
|
| 687 |
+
repo_type="space",
|
| 688 |
+
folder_path=str(dist_dir),
|
| 689 |
+
),
|
| 690 |
+
)
|
| 691 |
+
|
| 692 |
+
config = {
|
| 693 |
+
"mode": "static",
|
| 694 |
+
"project": project,
|
| 695 |
+
"private": bool(private),
|
| 696 |
+
}
|
| 697 |
+
if bucket_id is not None:
|
| 698 |
+
config["bucket_id"] = bucket_id
|
| 699 |
+
if dataset_id is not None:
|
| 700 |
+
config["dataset_id"] = dataset_id
|
| 701 |
+
if hf_token and private:
|
| 702 |
+
config["hf_token"] = hf_token
|
| 703 |
+
|
| 704 |
+
_retry_hf_write(
|
| 705 |
+
"Static Space config upload",
|
| 706 |
+
lambda: hf_api.upload_file(
|
| 707 |
+
path_or_fileobj=io.BytesIO(json_mod.dumps(config).encode("utf-8")),
|
| 708 |
+
path_in_repo="config.json",
|
| 709 |
+
repo_id=space_id,
|
| 710 |
+
repo_type="space",
|
| 711 |
+
),
|
| 712 |
+
)
|
| 713 |
+
|
| 714 |
+
assets_dir = Path(trackio.__file__).resolve().parent / "assets"
|
| 715 |
+
if assets_dir.is_dir():
|
| 716 |
+
_retry_hf_write(
|
| 717 |
+
"Static Space assets upload",
|
| 718 |
+
lambda: hf_api.upload_folder(
|
| 719 |
+
repo_id=space_id,
|
| 720 |
+
repo_type="space",
|
| 721 |
+
folder_path=str(assets_dir),
|
| 722 |
+
path_in_repo="assets",
|
| 723 |
+
),
|
| 724 |
+
)
|
| 725 |
+
|
| 726 |
+
print(
|
| 727 |
+
f"* Static Space deployed: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
|
| 728 |
+
)
|
| 729 |
+
|
| 730 |
+
|
| 731 |
+
def sync(
|
| 732 |
+
project: str,
|
| 733 |
+
space_id: str | None = None,
|
| 734 |
+
private: bool | None = None,
|
| 735 |
+
force: bool = False,
|
| 736 |
+
run_in_background: bool = False,
|
| 737 |
+
sdk: str = "gradio",
|
| 738 |
+
dataset_id: str | None = None,
|
| 739 |
+
bucket_id: str | None = None,
|
| 740 |
+
) -> str:
|
| 741 |
+
"""
|
| 742 |
+
Syncs a local Trackio project's database to a Hugging Face Space.
|
| 743 |
+
If the Space does not exist, it will be created. Local data is never deleted.
|
| 744 |
+
|
| 745 |
+
**Freezing:** Passing ``sdk="static"`` deploys a static Space backed by an HF Bucket
|
| 746 |
+
(read-only dashboard, no Gradio server). You can sync the same project again later to
|
| 747 |
+
refresh that static Space. If you want a one-time snapshot of an existing Gradio Space,
|
| 748 |
+
use ``freeze()`` instead.
|
| 749 |
+
|
| 750 |
+
Args:
|
| 751 |
+
project (`str`): The name of the project to upload.
|
| 752 |
+
space_id (`str`, *optional*): The ID of the Space to upload to (e.g., `"username/space_id"`).
|
| 753 |
+
If not provided, checks project metadata first, then generates a random space_id.
|
| 754 |
+
private (`bool`, *optional*):
|
| 755 |
+
Whether to make the Space private. If None (default), the repo will be
|
| 756 |
+
public unless the organization's default is private. This value is ignored
|
| 757 |
+
if the repo already exists.
|
| 758 |
+
force (`bool`, *optional*, defaults to `False`):
|
| 759 |
+
If `True`, overwrite the existing database without prompting for confirmation.
|
| 760 |
+
If `False`, prompt the user before overwriting an existing database.
|
| 761 |
+
run_in_background (`bool`, *optional*, defaults to `False`):
|
| 762 |
+
If `True`, the Space creation and database upload will be run in a background thread.
|
| 763 |
+
If `False`, all the steps will be run synchronously.
|
| 764 |
+
sdk (`str`, *optional*, defaults to `"gradio"`):
|
| 765 |
+
The type of Space to deploy. `"gradio"` deploys a Gradio Space with a live
|
| 766 |
+
server. `"static"` freezes the Space: deploys a static Space that reads from an HF Bucket
|
| 767 |
+
(no server needed).
|
| 768 |
+
dataset_id (`str`, *optional*):
|
| 769 |
+
Deprecated. Use `bucket_id` instead.
|
| 770 |
+
bucket_id (`str`, *optional*):
|
| 771 |
+
The ID of the HF Bucket to sync to. By default, a bucket is auto-generated
|
| 772 |
+
from the space_id.
|
| 773 |
+
Returns:
|
| 774 |
+
`str`: The Space ID of the synced project.
|
| 775 |
+
"""
|
| 776 |
+
if sdk not in ("gradio", "static"):
|
| 777 |
+
raise ValueError(f"sdk must be 'gradio' or 'static', got '{sdk}'")
|
| 778 |
+
if space_id is None:
|
| 779 |
+
space_id = SQLiteStorage.get_space_id(project)
|
| 780 |
+
if space_id is None:
|
| 781 |
+
space_id = f"{project}-{get_or_create_project_hash(project)}"
|
| 782 |
+
space_id, dataset_id, bucket_id = preprocess_space_and_dataset_ids(
|
| 783 |
+
space_id, dataset_id, bucket_id
|
| 784 |
+
)
|
| 785 |
+
|
| 786 |
+
def _do_sync():
|
| 787 |
+
try:
|
| 788 |
+
info = huggingface_hub.HfApi().space_info(space_id)
|
| 789 |
+
existing_sdk = info.sdk
|
| 790 |
+
if existing_sdk and existing_sdk != sdk:
|
| 791 |
+
raise ValueError(
|
| 792 |
+
f"Space '{space_id}' is a '{existing_sdk}' Space but sdk='{sdk}' was requested. "
|
| 793 |
+
f"The sdk must match the existing Space type."
|
| 794 |
+
)
|
| 795 |
+
except RepositoryNotFoundError:
|
| 796 |
+
pass
|
| 797 |
+
|
| 798 |
+
if sdk == "static":
|
| 799 |
+
if dataset_id is not None:
|
| 800 |
+
upload_dataset_for_static(project, dataset_id, private=private)
|
| 801 |
+
hf_token = huggingface_hub.utils.get_token() if private else None
|
| 802 |
+
deploy_as_static_space(
|
| 803 |
+
space_id,
|
| 804 |
+
dataset_id,
|
| 805 |
+
project,
|
| 806 |
+
private=private,
|
| 807 |
+
hf_token=hf_token,
|
| 808 |
+
)
|
| 809 |
+
elif bucket_id is not None:
|
| 810 |
+
create_bucket_if_not_exists(bucket_id, private=private)
|
| 811 |
+
upload_project_to_bucket_for_static(project, bucket_id)
|
| 812 |
+
print(
|
| 813 |
+
f"* Project data uploaded to bucket: https://huggingface.co/buckets/{bucket_id}"
|
| 814 |
+
)
|
| 815 |
+
deploy_as_static_space(
|
| 816 |
+
space_id,
|
| 817 |
+
None,
|
| 818 |
+
project,
|
| 819 |
+
bucket_id=bucket_id,
|
| 820 |
+
private=private,
|
| 821 |
+
hf_token=huggingface_hub.utils.get_token() if private else None,
|
| 822 |
+
)
|
| 823 |
+
else:
|
| 824 |
+
if bucket_id is not None:
|
| 825 |
+
create_bucket_if_not_exists(bucket_id, private=private)
|
| 826 |
+
upload_project_to_bucket(project, bucket_id)
|
| 827 |
+
print(
|
| 828 |
+
f"* Project data uploaded to bucket: https://huggingface.co/buckets/{bucket_id}"
|
| 829 |
+
)
|
| 830 |
+
create_space_if_not_exists(
|
| 831 |
+
space_id, bucket_id=bucket_id, private=private
|
| 832 |
+
)
|
| 833 |
+
else:
|
| 834 |
+
sync_incremental(project, space_id, private=private, pending_only=False)
|
| 835 |
+
SQLiteStorage.set_project_metadata(project, "space_id", space_id)
|
| 836 |
+
|
| 837 |
+
if run_in_background:
|
| 838 |
+
threading.Thread(target=_do_sync).start()
|
| 839 |
+
else:
|
| 840 |
+
_do_sync()
|
| 841 |
+
return space_id
|
| 842 |
+
|
| 843 |
+
|
| 844 |
+
def _get_source_bucket(space_id: str) -> str:
|
| 845 |
+
volumes = _get_space_volumes(space_id)
|
| 846 |
+
for v in volumes:
|
| 847 |
+
if v.type == "bucket" and v.mount_path == "/data":
|
| 848 |
+
return v.source
|
| 849 |
+
raise ValueError(
|
| 850 |
+
f"Space '{space_id}' has no bucket mounted at '/data'. "
|
| 851 |
+
f"freeze() requires the source Space to use bucket storage."
|
| 852 |
+
)
|
| 853 |
+
|
| 854 |
+
|
| 855 |
+
def freeze(
|
| 856 |
+
space_id: str,
|
| 857 |
+
project: str,
|
| 858 |
+
new_space_id: str | None = None,
|
| 859 |
+
private: bool | None = None,
|
| 860 |
+
bucket_id: str | None = None,
|
| 861 |
+
) -> str:
|
| 862 |
+
"""
|
| 863 |
+
Creates a new static Hugging Face Space containing a read-only snapshot of
|
| 864 |
+
the data for the specified project from the source Gradio Space. The data is
|
| 865 |
+
read from the bucket attached to the source Space at freeze time. The original
|
| 866 |
+
Space is not modified, and the new static Space does not automatically reflect
|
| 867 |
+
metrics uploaded to the original Gradio Space after the freeze completes.
|
| 868 |
+
|
| 869 |
+
Args:
|
| 870 |
+
space_id (`str`):
|
| 871 |
+
The ID of the source Gradio Space (e.g., `"username/my-space"` or a
|
| 872 |
+
short repo name with the logged-in namespace inferred, like `init()`).
|
| 873 |
+
Must be a Gradio Space with a bucket mounted at `/data`.
|
| 874 |
+
project (`str`):
|
| 875 |
+
The name of the project whose data to include in the frozen Space.
|
| 876 |
+
new_space_id (`str`, *optional*):
|
| 877 |
+
The ID for the new static Space. If not provided, defaults to
|
| 878 |
+
`"{space_id}_static"`.
|
| 879 |
+
private (`bool`, *optional*):
|
| 880 |
+
Whether to make the new Space private. If None (default), the repo
|
| 881 |
+
will be public unless the organization's default is private.
|
| 882 |
+
bucket_id (`str`, *optional*):
|
| 883 |
+
The ID of the HF Bucket for the new static Space's data storage.
|
| 884 |
+
If not provided, one is auto-generated from the new Space ID.
|
| 885 |
+
|
| 886 |
+
Returns:
|
| 887 |
+
`str`: The Space ID of the newly created static Space.
|
| 888 |
+
"""
|
| 889 |
+
space_id, _, _ = preprocess_space_and_dataset_ids(space_id, None, None)
|
| 890 |
+
|
| 891 |
+
try:
|
| 892 |
+
info = huggingface_hub.HfApi().space_info(space_id)
|
| 893 |
+
if info.sdk != "gradio":
|
| 894 |
+
raise ValueError(
|
| 895 |
+
f"Space '{space_id}' is not a Gradio Space (sdk='{info.sdk}'). "
|
| 896 |
+
f"freeze() requires a Gradio Space as the source."
|
| 897 |
+
)
|
| 898 |
+
except RepositoryNotFoundError:
|
| 899 |
+
raise ValueError(
|
| 900 |
+
f"Space '{space_id}' not found. Provide an existing Gradio Space ID."
|
| 901 |
+
)
|
| 902 |
+
|
| 903 |
+
source_bucket_id = _get_source_bucket(space_id)
|
| 904 |
+
print(f"* Reading project data from bucket: {source_bucket_id}")
|
| 905 |
+
|
| 906 |
+
if new_space_id is None:
|
| 907 |
+
new_space_id = f"{space_id}_static"
|
| 908 |
+
new_space_id, _dataset_id, bucket_id = preprocess_space_and_dataset_ids(
|
| 909 |
+
new_space_id, None, bucket_id
|
| 910 |
+
)
|
| 911 |
+
|
| 912 |
+
hf_api = huggingface_hub.HfApi()
|
| 913 |
+
try:
|
| 914 |
+
dest_info = hf_api.space_info(new_space_id)
|
| 915 |
+
tags = dest_info.tags or []
|
| 916 |
+
if dest_info.sdk != "static" or "trackio" not in tags:
|
| 917 |
+
raise ValueError(
|
| 918 |
+
f"Space '{new_space_id}' already exists and is not a Trackio static Space "
|
| 919 |
+
f"(sdk='{dest_info.sdk}', tags={tags}). Choose a different new_space_id "
|
| 920 |
+
f"or delete the existing Space first."
|
| 921 |
+
)
|
| 922 |
+
except RepositoryNotFoundError:
|
| 923 |
+
pass
|
| 924 |
+
|
| 925 |
+
create_bucket_if_not_exists(bucket_id, private=private)
|
| 926 |
+
export_from_bucket_for_static(source_bucket_id, bucket_id, project)
|
| 927 |
+
print(
|
| 928 |
+
f"* Project data uploaded to bucket: https://huggingface.co/buckets/{bucket_id}"
|
| 929 |
+
)
|
| 930 |
+
deploy_as_static_space(
|
| 931 |
+
new_space_id,
|
| 932 |
+
None,
|
| 933 |
+
project,
|
| 934 |
+
bucket_id=bucket_id,
|
| 935 |
+
private=private,
|
| 936 |
+
hf_token=huggingface_hub.utils.get_token() if private else None,
|
| 937 |
+
)
|
| 938 |
+
return new_space_id
|
trackio/dummy_commit_scheduler.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from concurrent.futures import Future
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class DummyCommitSchedulerLock:
|
| 5 |
+
def __enter__(self):
|
| 6 |
+
return None
|
| 7 |
+
|
| 8 |
+
def __exit__(self, exception_type, exception_value, exception_traceback):
|
| 9 |
+
pass
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class DummyCommitScheduler:
|
| 13 |
+
def __init__(self):
|
| 14 |
+
self.lock = DummyCommitSchedulerLock()
|
| 15 |
+
|
| 16 |
+
def trigger(self) -> Future:
|
| 17 |
+
fut: Future = Future()
|
| 18 |
+
fut.set_result(None)
|
| 19 |
+
return fut
|
trackio/exceptions.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
class TrackioAPIError(Exception):
|
| 2 |
+
pass
|
trackio/frontend/dist/assets/index-Cs1UkfOV.css
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
:root{--primary-50: #fff7ed;--primary-100: #ffedd5;--primary-200: #fed7aa;--primary-300: #fdba74;--primary-400: #fb923c;--primary-500: #f97316;--primary-600: #ea580c;--primary-700: #c2410c;--primary-800: #9a3412;--primary-900: #7c2d12;--primary-950: #6c2e12;--secondary-50: #eff6ff;--secondary-100: #dbeafe;--secondary-200: #bfdbfe;--secondary-300: #93c5fd;--secondary-400: #60a5fa;--secondary-500: #3b82f6;--secondary-600: #2563eb;--secondary-700: #1d4ed8;--secondary-800: #1e40af;--secondary-900: #1e3a8a;--secondary-950: #1d3660;--neutral-50: #f9fafb;--neutral-100: #f3f4f6;--neutral-200: #e5e7eb;--neutral-300: #d1d5db;--neutral-400: #9ca3af;--neutral-500: #6b7280;--neutral-600: #4b5563;--neutral-700: #374151;--neutral-800: #1f2937;--neutral-900: #111827;--neutral-950: #0b0f19;--size-0-5: 2px;--size-1: 4px;--size-2: 8px;--size-3: 12px;--size-4: 16px;--size-5: 20px;--size-6: 24px;--size-8: 32px;--size-14: 56px;--size-16: 64px;--size-28: 112px;--size-full: 100%;--spacing-xxs: 1px;--spacing-xs: 2px;--spacing-sm: 4px;--spacing-md: 6px;--spacing-lg: 8px;--spacing-xl: 10px;--spacing-xxl: 16px;--radius-xxs: 1px;--radius-xs: 2px;--radius-sm: 3px;--radius-md: 4px;--radius-lg: 5px;--radius-xl: 8px;--radius-xxl: 12px;--text-xxs: 9px;--text-xs: 10px;--text-sm: 12px;--text-md: 14px;--text-lg: 16px;--text-xl: 22px;--text-xxl: 26px;--line-sm: 1.4;--background-fill-primary: white;--background-fill-secondary: var(--neutral-50);--body-text-color: var(--neutral-900);--body-text-color-subdued: var(--neutral-600);--border-color-primary: var(--neutral-200);--color-accent: var(--primary-500);--color-accent-soft: var(--primary-50);--shadow-drop: rgba(0, 0, 0, .05) 0px 1px 2px 0px;--shadow-drop-lg: 0 1px 3px 0 rgb(0 0 0 / .1), 0 1px 2px -1px rgb(0 0 0 / .1);--shadow-inset: rgba(0, 0, 0, .05) 0px 2px 4px 0px inset;--shadow-spread: 3px;--block-title-text-color: var(--neutral-500);--block-title-text-size: var(--text-md);--block-title-text-weight: 400;--block-info-text-color: var(--body-text-color-subdued);--block-info-text-size: var(--text-sm);--input-background-fill: white;--input-background-fill-focus: var(--primary-500);--input-border-color: var(--border-color-primary);--input-border-color-focus: var(--primary-300);--input-border-width: 1px;--input-padding: var(--spacing-xl);--input-placeholder-color: var(--neutral-400);--input-radius: var(--radius-lg);--input-shadow: 0 0 0 var(--shadow-spread) transparent, var(--shadow-inset);--input-shadow-focus: 0 0 0 var(--shadow-spread) var(--primary-50), var(--shadow-inset);--input-text-size: var(--text-md);--checkbox-background-color: var(--background-fill-primary);--checkbox-background-color-focus: var(--checkbox-background-color);--checkbox-background-color-hover: var(--checkbox-background-color);--checkbox-background-color-selected: var(--primary-600);--checkbox-border-color: var(--neutral-300);--checkbox-border-color-focus: var(--primary-500);--checkbox-border-color-hover: var(--neutral-300);--checkbox-border-color-selected: var(--primary-600);--checkbox-border-radius: var(--radius-sm);--checkbox-border-width: var(--input-border-width);--checkbox-label-gap: var(--spacing-lg);--checkbox-label-padding: var(--spacing-md) calc(2 * var(--spacing-md));--checkbox-label-text-size: var(--text-md);--checkbox-shadow: var(--input-shadow);--checkbox-check: url("data:image/svg+xml,%3csvg viewBox='0 0 16 16' fill='white' xmlns='http://www.w3.org/2000/svg'%3e%3cpath d='M12.207 4.793a1 1 0 010 1.414l-5 5a1 1 0 01-1.414 0l-2-2a1 1 0 011.414-1.414L6.5 9.086l4.293-4.293a1 1 0 011.414 0z'/%3e%3c/svg%3e");--slider-color: var(--primary-500);--container-radius: var(--radius-lg);--layer-top: 9999}.navbar.svelte-d8j1hi{display:flex;align-items:stretch;border-bottom:1px solid var(--border-color-primary, #e5e7eb);background:var(--background-fill-primary, white);padding:0;flex-shrink:0;min-height:44px}.nav-spacer.svelte-d8j1hi{flex:1}.nav-tabs.svelte-d8j1hi{display:flex;gap:0;padding-right:16px}.nav-link.svelte-d8j1hi{padding:10px 16px;border:none;background:none;color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-md, 14px);cursor:pointer;white-space:nowrap;border-bottom:2px solid transparent;transition:color .15s;font-weight:400}.nav-link.svelte-d8j1hi:hover{color:var(--body-text-color, #1f2937)}.nav-link.active.svelte-d8j1hi{color:var(--body-text-color, #1f2937);border-bottom-color:var(--body-text-color, #1f2937);font-weight:500}.checkbox-group.svelte-17gmtkf{display:flex;flex-direction:column}.checkbox-item.svelte-17gmtkf{display:flex;align-items:center;gap:8px;padding:3px 0;cursor:pointer;font-size:13px}.checkbox-item.svelte-17gmtkf input[type=checkbox]:where(.svelte-17gmtkf){-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;margin:0;border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);box-shadow:var(--checkbox-shadow);cursor:pointer;flex-shrink:0;transition:background-color .15s,border-color .15s}.checkbox-item.svelte-17gmtkf input[type=checkbox]:where(.svelte-17gmtkf):checked{background-image:var(--checkbox-check);background-color:var(--checkbox-background-color-selected, #f97316);border-color:var(--checkbox-border-color-selected, #f97316)}.checkbox-item.svelte-17gmtkf input[type=checkbox]:where(.svelte-17gmtkf):hover{border-color:var(--checkbox-border-color-hover, #d1d5db)}.color-dot.svelte-17gmtkf{width:10px;height:10px;border-radius:50%;flex-shrink:0}.run-name.svelte-17gmtkf{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:var(--body-text-color, #1f2937)}.dropdown-container.svelte-kgylqb{width:100%}.label.svelte-kgylqb{display:block;font-size:13px;font-weight:500;color:var(--body-text-color-subdued, #6b7280);margin-bottom:6px}.info.svelte-kgylqb{display:block;font-size:12px;color:var(--body-text-color-subdued, #9ca3af);margin-bottom:4px}.wrap.svelte-kgylqb{position:relative;border-radius:var(--input-radius, 8px);background:var(--input-background-fill, white);border:1px solid var(--border-color-primary, #e5e7eb);transition:border-color .15s,box-shadow .15s}.wrap.focused.svelte-kgylqb{border-color:var(--input-border-color-focus, #fdba74);box-shadow:0 0 0 2px var(--primary-50, #fff7ed)}.wrap-inner.svelte-kgylqb{display:flex;position:relative;align-items:center;padding:0 10px}.secondary-wrap.svelte-kgylqb{display:flex;flex:1;align-items:center}input.svelte-kgylqb{margin:0;outline:none;border:none;background:inherit;width:100%;color:var(--body-text-color, #1f2937);font-size:13px;font-family:inherit;padding:7px 0}input.svelte-kgylqb::placeholder{color:var(--input-placeholder-color, #9ca3af)}input[readonly].svelte-kgylqb{cursor:pointer}.icon-wrap.svelte-kgylqb{color:var(--body-text-color-subdued, #9ca3af);width:16px;flex-shrink:0;pointer-events:none}.options.svelte-kgylqb{position:fixed;z-index:var(--layer-top, 9999);margin:0;padding:4px 0;box-shadow:0 4px 12px #0000001f;border-radius:var(--input-radius, 8px);border:1px solid var(--border-color-primary, #e5e7eb);background:var(--background-fill-primary, white);min-width:fit-content;overflow:auto;color:var(--body-text-color, #1f2937);list-style:none}.item.svelte-kgylqb{display:flex;cursor:pointer;padding:6px 10px;font-size:13px;word-break:break-word}.item.svelte-kgylqb:hover,.item.active.svelte-kgylqb{background:var(--background-fill-secondary, #f9fafb)}.item.selected.svelte-kgylqb{font-weight:500}.check-mark.svelte-kgylqb{padding-right:6px;min-width:16px;font-size:12px}.check-mark.hide.svelte-kgylqb{visibility:hidden}.checkbox-container.svelte-oj84db{display:flex;align-items:center;gap:8px;cursor:pointer;margin:8px 0}.label-text.svelte-oj84db{color:var(--body-text-color, #1f2937);font-size:13px;line-height:1.4}input[type=checkbox].svelte-oj84db{--ring-color: transparent;position:relative;-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;box-shadow:var(--checkbox-shadow);border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);flex-shrink:0;cursor:pointer;transition:background-color .15s,border-color .15s}input[type=checkbox].svelte-oj84db:checked,input[type=checkbox].svelte-oj84db:checked:hover,input[type=checkbox].svelte-oj84db:checked:focus{background-image:var(--checkbox-check);background-color:var(--checkbox-background-color-selected, #f97316);border-color:var(--checkbox-border-color-selected, #f97316)}input[type=checkbox].svelte-oj84db:hover{border-color:var(--checkbox-border-color-hover, #d1d5db);background-color:var(--checkbox-background-color-hover, white)}input[type=checkbox].svelte-oj84db:focus{border-color:var(--checkbox-border-color-focus, #f97316);background-color:var(--checkbox-background-color-focus, white);outline:none}.slider-wrap.svelte-wei6ev{display:flex;flex-direction:column;width:100%}.head.svelte-wei6ev{margin-bottom:4px;display:flex;justify-content:space-between;align-items:center;width:100%}.label.svelte-wei6ev{flex:1;font-size:13px;font-weight:500;color:var(--body-text-color-subdued, #6b7280)}.info.svelte-wei6ev{display:block;font-size:12px;color:var(--body-text-color-subdued, #9ca3af);margin-bottom:4px}.slider-input-container.svelte-wei6ev{display:flex;align-items:center;gap:6px}input[type=range].svelte-wei6ev{-webkit-appearance:none;-moz-appearance:none;appearance:none;width:100%;cursor:pointer;outline:none;border-radius:var(--radius-xl, 12px);min-width:var(--size-28, 112px);background:transparent}input[type=range].svelte-wei6ev::-webkit-slider-runnable-track{height:6px;border-radius:var(--radius-xl, 12px);background:linear-gradient(to right,var(--slider-color, #f97316) var(--range_progress, 50%),var(--neutral-200, #e5e7eb) var(--range_progress, 50%))}input[type=range].svelte-wei6ev::-webkit-slider-thumb{-webkit-appearance:none;-moz-appearance:none;appearance:none;height:16px;width:16px;background-color:var(--slider-color, #f97316);border:2px solid var(--background-fill-primary, white);border-radius:50%;margin-top:-5px;box-shadow:0 0 0 1px var(--border-color-primary, rgba(0, 0, 0, .08)),0 1px 3px #0003}input[type=range].svelte-wei6ev::-moz-range-track{height:6px;background:var(--neutral-200, #e5e7eb);border-radius:var(--radius-xl, 12px)}input[type=range].svelte-wei6ev::-moz-range-thumb{-webkit-appearance:none;-moz-appearance:none;appearance:none;height:16px;width:16px;background-color:var(--slider-color, #f97316);border:2px solid var(--background-fill-primary, white);border-radius:50%;box-shadow:0 0 0 1px var(--border-color-primary, rgba(0, 0, 0, .08)),0 1px 3px #0003}input[type=range].svelte-wei6ev::-moz-range-progress{height:6px;background-color:var(--slider-color, #f97316);border-radius:var(--radius-xl, 12px)}.bound.svelte-wei6ev{font-size:11px;color:var(--body-text-color-subdued, #9ca3af);min-width:12px;text-align:center}.textbox-container.svelte-6yncpg{width:100%}.label.svelte-6yncpg{display:block;font-size:13px;font-weight:500;color:var(--body-text-color-subdued, #6b7280);margin-bottom:6px}.info.svelte-6yncpg{display:block;font-size:12px;color:var(--body-text-color-subdued, #9ca3af);margin-bottom:4px}.input-wrap.svelte-6yncpg{border-radius:var(--input-radius, 8px);background:var(--input-background-fill, white);border:1px solid var(--border-color-primary, #e5e7eb);transition:border-color .15s,box-shadow .15s}.input-wrap.svelte-6yncpg:focus-within{border-color:var(--input-border-color-focus, #fdba74);box-shadow:0 0 0 2px var(--primary-50, #fff7ed)}input.svelte-6yncpg{width:100%;padding:7px 10px;outline:none;border:none;background:transparent;color:var(--body-text-color, #1f2937);font-size:13px;font-family:inherit;border-radius:var(--input-radius, 8px)}input.svelte-6yncpg::placeholder{color:var(--input-placeholder-color, #9ca3af)}.sidebar.svelte-181dlmc{width:290px;min-width:290px;background:var(--background-fill-primary, white);border-right:1px solid var(--border-color-primary, #e5e7eb);display:flex;flex-direction:column;position:relative;overflow:hidden;transition:width .2s,min-width .2s}.sidebar.collapsed.svelte-181dlmc{width:40px;min-width:40px}.toggle-btn.svelte-181dlmc{position:absolute;top:12px;right:8px;z-index:10;border:none;background:none;color:var(--body-text-color-subdued, #9ca3af);cursor:pointer;padding:4px;display:flex;align-items:center;justify-content:center;border-radius:var(--radius-sm, 4px);transition:color .15s,background-color .15s}.toggle-btn.svelte-181dlmc:hover{color:var(--body-text-color, #1f2937);background-color:var(--background-fill-secondary, #f9fafb)}.sidebar-content.svelte-181dlmc{padding:16px;flex:1;min-height:0;display:flex;flex-direction:column}.sidebar-scroll.svelte-181dlmc{overflow-y:auto;flex:1;min-height:0}.oauth-footer.svelte-181dlmc{flex-shrink:0;margin-top:12px;padding-top:12px;border-top:1px solid var(--border-color-primary, #e5e7eb)}.readonly-footer.svelte-181dlmc{flex-shrink:0;margin-top:12px;padding-top:12px;border-top:1px solid var(--border-color-primary, #e5e7eb);display:flex;align-items:center;gap:8px;flex-wrap:wrap}.readonly-badge.svelte-181dlmc{display:inline-flex;align-items:center;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:999px;padding:2px 8px;font-size:10px;letter-spacing:.06em;font-weight:600;color:var(--body-text-color-subdued, #6b7280);background:var(--background-fill-secondary, #f9fafb)}.readonly-link.svelte-181dlmc{font-size:12px;color:var(--body-text-color-subdued, #6b7280);text-decoration:none;max-width:100%;overflow-wrap:anywhere}.readonly-link.svelte-181dlmc:hover{color:var(--body-text-color, #1f2937);text-decoration:underline}.oauth-line.svelte-181dlmc{margin:0;font-size:12px;line-height:1.4;color:var(--body-text-color-subdued, #6b7280)}.oauth-warn.svelte-181dlmc{color:var(--body-text-color, #92400e)}.hf-login-btn.svelte-181dlmc{display:inline-flex;align-items:center;justify-content:center;gap:8px;width:100%;padding:8px 12px;font-size:13px;font-weight:600;color:#fff;background:#141c2e;border-radius:var(--radius-lg, 8px);text-decoration:none;border:none;cursor:pointer;box-sizing:border-box}.hf-login-btn.svelte-181dlmc:hover{background:#283042}.hf-logo.svelte-181dlmc{width:20px;height:20px;flex-shrink:0}.oauth-hint.svelte-181dlmc{margin:8px 0 0;font-size:11px;line-height:1.35;color:var(--body-text-color-subdued, #9ca3af)}.oauth-signed-in.svelte-181dlmc{margin:0;font-size:12px;color:var(--body-text-color-subdued, #6b7280)}.oauth-logout.svelte-181dlmc{font-size:12px;color:var(--body-text-color-subdued, #9ca3af);text-decoration:none;cursor:pointer}.oauth-logout.svelte-181dlmc:hover{text-decoration:underline;color:var(--body-text-color, #1f2937)}.logo-section.svelte-181dlmc{margin-bottom:20px}.logo.svelte-181dlmc{width:80%;max-width:200px}.section.svelte-181dlmc{margin-bottom:18px}.section-label.svelte-181dlmc{font-size:13px;font-weight:500;color:var(--body-text-color-subdued, #6b7280)}.locked-project.svelte-181dlmc{margin-top:4px;font-size:13px;font-weight:500;color:var(--body-text-color, #1f2937);padding:8px 10px;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-md, 6px);background:var(--background-fill-secondary, #f9fafb)}.runs-header.svelte-181dlmc{display:flex;align-items:center;justify-content:space-between;margin-bottom:6px}.select-all-label.svelte-181dlmc{display:flex;align-items:center;gap:6px;cursor:pointer}.select-all-cb.svelte-181dlmc{-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);cursor:pointer;flex-shrink:0;position:relative;transition:background-color .15s,border-color .15s}.select-all-cb.svelte-181dlmc:checked{background-color:var(--checkbox-background-color-selected, var(--color-accent, #f97316));border-color:var(--checkbox-background-color-selected, var(--color-accent, #f97316));background-image:var(--checkbox-check)}.select-all-cb.svelte-181dlmc:indeterminate{background-color:var(--checkbox-background-color-selected, var(--color-accent, #f97316));border-color:var(--checkbox-background-color-selected, var(--color-accent, #f97316));background-image:url("data:image/svg+xml,%3Csvg viewBox='0 0 16 16' fill='white' xmlns='http://www.w3.org/2000/svg'%3E%3Crect x='3' y='7' width='10' height='2' rx='1'/%3E%3C/svg%3E");background-size:12px;background-position:center;background-repeat:no-repeat}.latest-toggle.svelte-181dlmc{display:flex;align-items:center;gap:6px;font-size:12px;color:var(--body-text-color-subdued, #6b7280);cursor:pointer}.latest-toggle.svelte-181dlmc input[type=checkbox]:where(.svelte-181dlmc){-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;margin:0;border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);box-shadow:var(--checkbox-shadow);cursor:pointer;flex-shrink:0;transition:background-color .15s,border-color .15s}.latest-toggle.svelte-181dlmc input[type=checkbox]:where(.svelte-181dlmc):checked{background-image:var(--checkbox-check);background-color:var(--checkbox-background-color-selected, #f97316);border-color:var(--checkbox-border-color-selected, #f97316)}.checkbox-list.svelte-181dlmc{max-height:300px;overflow-y:auto;margin-top:8px}.alert-panel.svelte-x5aqew{position:fixed;bottom:16px;right:16px;width:380px;max-height:400px;background:var(--background-fill-primary, white);border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);box-shadow:var(--shadow-drop-lg);z-index:1000;overflow:hidden;display:flex;flex-direction:column}.alert-panel.collapsed.svelte-x5aqew{max-height:none}.alert-header.svelte-x5aqew{padding:10px 12px;border:none;border-bottom:1px solid var(--border-color-primary, #e5e7eb);background:none;width:100%;display:flex;align-items:center;justify-content:space-between;cursor:pointer;gap:8px}.alert-panel.collapsed.svelte-x5aqew .alert-header:where(.svelte-x5aqew){border-bottom:none}.collapse-icon.svelte-x5aqew{color:var(--body-text-color-subdued, #9ca3af);flex-shrink:0;transition:transform .15s}.collapse-icon.rotated.svelte-x5aqew{transform:rotate(-90deg)}.alert-title.svelte-x5aqew{font-size:13px;font-weight:600;color:var(--body-text-color, #1f2937)}.filter-pills.svelte-x5aqew{display:flex;gap:4px}.pill.svelte-x5aqew{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-xxl, 22px);padding:2px 8px;font-size:11px;background:var(--background-fill-secondary, #f9fafb);color:var(--body-text-color-subdued, #6b7280);cursor:pointer}.pill.active.svelte-x5aqew{background:var(--color-accent, #f97316);color:#fff;border-color:var(--color-accent, #f97316)}.alert-list.svelte-x5aqew{overflow-y:auto;flex:1}.alert-item.svelte-x5aqew{border-bottom:1px solid var(--neutral-100, #f3f4f6)}.alert-row.svelte-x5aqew{display:flex;align-items:center;gap:8px;width:100%;padding:8px 12px;border:none;background:none;text-align:left;cursor:pointer;font-size:var(--text-sm, 12px)}.alert-row.svelte-x5aqew:hover{background:var(--background-fill-secondary, #f9fafb)}.alert-text.svelte-x5aqew{flex:1;color:var(--body-text-color, #1f2937)}.alert-meta.svelte-x5aqew{font-size:var(--text-xs, 10px);color:var(--body-text-color-subdued, #9ca3af);white-space:nowrap}.alert-detail.svelte-x5aqew{padding:4px 12px 8px 32px;font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #6b7280)}.plot-container.svelte-9thu1j{min-width:350px;flex:1;background:var(--background-fill-primary, white);border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);padding:12px;overflow:hidden;position:relative}.plot-container[draggable=true].svelte-9thu1j{cursor:grab}.plot-container[draggable=true].svelte-9thu1j:active{cursor:grabbing}.hidden-plot.svelte-9thu1j{visibility:hidden;height:0;padding:0;margin:0;border:none;overflow:hidden;pointer-events:none}.drag-handle.svelte-9thu1j{position:absolute;top:8px;left:8px;color:var(--body-text-color-subdued, #9ca3af);opacity:0;transition:opacity .15s;z-index:5}.plot-container.svelte-9thu1j:hover .drag-handle:where(.svelte-9thu1j){opacity:.5}.drag-handle.svelte-9thu1j:hover{opacity:1!important}.plot-toolbar.svelte-9thu1j{position:absolute;top:8px;right:8px;display:flex;gap:4px;z-index:5;opacity:0;transition:opacity .15s}.plot-container.svelte-9thu1j:hover .plot-toolbar:where(.svelte-9thu1j){opacity:1}.toolbar-btn.svelte-9thu1j{border:1px solid var(--border-color-primary, #e5e7eb);background:var(--background-fill-primary, white);color:var(--body-text-color-subdued, #6b7280);cursor:pointer;padding:4px 6px;border-radius:var(--radius-sm, 4px);display:flex;align-items:center;justify-content:center}.toolbar-btn.svelte-9thu1j:hover{background:var(--neutral-100, #f3f4f6);color:var(--body-text-color, #1f2937)}.plot-chart-wrap.svelte-9thu1j{position:relative;width:100%}.plot-chart-wrap--fs.svelte-9thu1j{flex:1;min-height:0;display:flex;flex-direction:column}.reset-zoom-btn.svelte-9thu1j{position:absolute;bottom:1px;right:1px;z-index:6;display:inline-flex;align-items:center;justify-content:center;margin:0;min-width:52px;padding:5px 12px 5px 10px;border:none;border-radius:4px;background:transparent;color:var(--body-text-color-subdued, #334155);cursor:pointer;opacity:.92;transform:translateY(6px);transition:opacity .15s ease,color .15s ease,background .15s ease;box-shadow:none}.reset-zoom-btn.svelte-9thu1j:hover{opacity:1;color:var(--body-text-color, #0f172a);background:var(--background-fill-secondary, rgba(226, 232, 240, .85));transform:translateY(6px)}.reset-zoom-btn.svelte-9thu1j svg:where(.svelte-9thu1j){display:block;flex-shrink:0;filter:drop-shadow(0 0 .5px rgba(255,255,255,.95))}.plot.svelte-9thu1j{width:100%}.plot.svelte-9thu1j .vega-embed{width:100%!important}.plot.svelte-9thu1j .vega-embed summary{display:none}.fullscreen-host.svelte-9thu1j{position:fixed;top:0;right:0;bottom:0;left:0;z-index:10000;box-sizing:border-box;display:flex;flex-direction:column;background:var(--background-fill-primary, white);padding:12px;gap:8px;pointer-events:auto}.fullscreen-host.svelte-9thu1j:fullscreen{width:100%;height:100%}.fullscreen-host.svelte-9thu1j:-webkit-full-screen{width:100%;height:100%}.fullscreen-toolbar.svelte-9thu1j{flex-shrink:0;display:flex;justify-content:flex-end;gap:4px;z-index:5}.fullscreen-chart-wrap.svelte-9thu1j{flex:1;min-height:0;display:flex;flex-direction:column}.fullscreen-legend.svelte-9thu1j{flex-shrink:0}.fullscreen-plot.svelte-9thu1j{flex:1;min-height:0;width:100%;overflow:hidden}.fullscreen-plot.svelte-9thu1j .vega-embed{width:100%!important;height:100%!important;min-height:0;display:flex;flex-direction:column}.fullscreen-plot.svelte-9thu1j .vega-embed .vega-view{flex:1;min-height:0}.fullscreen-plot.svelte-9thu1j .vega-embed summary{display:none}.custom-legend.svelte-9thu1j{display:flex;align-items:center;justify-content:center;gap:12px;padding:6px 0 0;flex-wrap:wrap}.legend-title.svelte-9thu1j{font-size:11px;color:var(--body-text-color-subdued, #6b7280);font-weight:600}.legend-item.svelte-9thu1j{display:flex;align-items:center;gap:4px}.legend-dot.svelte-9thu1j{width:10px;height:10px;border-radius:50%;flex-shrink:0}.legend-label.svelte-9thu1j{font-size:11px;color:var(--body-text-color-subdued, #6b7280)}.plot-container.svelte-1swghqy{min-width:350px;flex:1;background:var(--background-fill-primary, white);border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);padding:12px;overflow:hidden;position:relative}.plot-container[draggable=true].svelte-1swghqy{cursor:grab}.plot-container[draggable=true].svelte-1swghqy:active{cursor:grabbing}.hidden-plot.svelte-1swghqy{visibility:hidden;height:0;padding:0;margin:0;border:none;overflow:hidden;pointer-events:none}.drag-handle.svelte-1swghqy{position:absolute;top:8px;left:8px;color:var(--body-text-color-subdued, #9ca3af);opacity:0;transition:opacity .15s;z-index:5}.plot-container.svelte-1swghqy:hover .drag-handle:where(.svelte-1swghqy){opacity:.5}.drag-handle.svelte-1swghqy:hover{opacity:1!important}.plot-toolbar.svelte-1swghqy{position:absolute;top:8px;right:8px;display:flex;gap:4px;z-index:5;opacity:0;transition:opacity .15s}.plot-container.svelte-1swghqy:hover .plot-toolbar:where(.svelte-1swghqy){opacity:1}.toolbar-btn.svelte-1swghqy{border:1px solid var(--border-color-primary, #e5e7eb);background:var(--background-fill-primary, white);color:var(--body-text-color-subdued, #6b7280);cursor:pointer;padding:4px 6px;border-radius:var(--radius-sm, 4px);display:flex;align-items:center;justify-content:center}.toolbar-btn.svelte-1swghqy:hover{background:var(--neutral-100, #f3f4f6);color:var(--body-text-color, #1f2937)}.plot-chart-wrap.svelte-1swghqy{position:relative;width:100%}.plot-chart-wrap--fs.svelte-1swghqy{flex:1;min-height:0;display:flex;flex-direction:column}.plot.svelte-1swghqy{width:100%}.plot.svelte-1swghqy .vega-embed{width:100%!important}.plot.svelte-1swghqy .vega-embed summary{display:none}.fullscreen-host.svelte-1swghqy{position:fixed;top:0;right:0;bottom:0;left:0;z-index:10000;box-sizing:border-box;display:flex;flex-direction:column;background:var(--background-fill-primary, white);padding:12px;gap:8px;pointer-events:auto}.fullscreen-host.svelte-1swghqy:fullscreen{width:100%;height:100%}.fullscreen-host.svelte-1swghqy:-webkit-full-screen{width:100%;height:100%}.fullscreen-toolbar.svelte-1swghqy{flex-shrink:0;display:flex;justify-content:flex-end;gap:4px;z-index:5}.fullscreen-chart-wrap.svelte-1swghqy{flex:1;min-height:0;display:flex;flex-direction:column}.fullscreen-legend.svelte-1swghqy{flex-shrink:0}.fullscreen-plot.svelte-1swghqy{flex:1;min-height:0;width:100%;overflow:hidden}.fullscreen-plot.svelte-1swghqy .vega-embed{width:100%!important;height:100%!important;min-height:0;display:flex;flex-direction:column}.fullscreen-plot.svelte-1swghqy .vega-embed .vega-view{flex:1;min-height:0}.fullscreen-plot.svelte-1swghqy .vega-embed summary{display:none}.custom-legend.svelte-1swghqy{display:flex;align-items:center;justify-content:center;gap:12px;padding:6px 0 0;flex-wrap:wrap}.legend-title.svelte-1swghqy{font-size:11px;color:var(--body-text-color-subdued, #6b7280);font-weight:600}.legend-item.svelte-1swghqy{display:flex;align-items:center;gap:4px}.legend-dot.svelte-1swghqy{width:10px;height:10px;border-radius:50%;flex-shrink:0}.legend-label.svelte-1swghqy{font-size:11px;color:var(--body-text-color-subdued, #6b7280)}.accordion.svelte-1jep0a{margin-bottom:12px;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);background:var(--background-fill-primary, white);overflow:hidden}.accordion-hidden.svelte-1jep0a{margin-bottom:8px}.accordion-header.svelte-1jep0a{display:flex;align-items:center;gap:8px;width:100%;padding:10px 14px;border:none;background:var(--background-fill-primary, white);color:var(--body-text-color, #1f2937);font-size:var(--text-md, 14px);font-weight:600;cursor:pointer;text-align:left}.accordion-header.svelte-1jep0a:hover{background:var(--background-fill-secondary, #f9fafb)}.arrow.svelte-1jep0a{font-size:14px;transition:transform .15s;color:var(--body-text-color, #1f2937);display:inline-block}.arrow.svelte-1jep0a:not(.rotated){transform:rotate(-90deg)}.accordion-body.svelte-1jep0a{padding:0 14px 14px}.trackio-loading.svelte-1kc6b2l{display:flex;align-items:center;justify-content:center;width:100%;min-height:min(70vh,640px);padding:32px 24px;box-sizing:border-box;background:transparent}.logo-stack.svelte-1kc6b2l{position:relative;width:min(100%,200px);max-width:min(92vw,200px);line-height:0;background:transparent;isolation:isolate}.logo-base.svelte-1kc6b2l{display:block;background:transparent}.logo-img.svelte-1kc6b2l{width:100%;height:auto;display:block;background:transparent}.logo-overlay.svelte-1kc6b2l{position:absolute;left:0;top:0;width:100%;animation:svelte-1kc6b2l-trackio-logo-sweep 4s linear infinite;pointer-events:none;background:transparent}.logo-overlay.svelte-1kc6b2l .logo-img:where(.svelte-1kc6b2l){width:100%;height:auto;object-position:left center}.logo-img--gray.svelte-1kc6b2l{filter:grayscale(1)}@keyframes svelte-1kc6b2l-trackio-logo-sweep{0%{clip-path:inset(0 0 0 0)}50%{clip-path:inset(0 0 0 100%)}to{clip-path:inset(0 0 0 0)}}@media(prefers-reduced-motion:reduce){.logo-overlay.svelte-1kc6b2l{display:none}}.sr-only.svelte-1kc6b2l{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.metrics-page.svelte-2bul55{padding:20px 24px;overflow-y:auto;flex:1;min-height:0}.plot-grid.svelte-2bul55{display:flex;flex-wrap:wrap;gap:16px}.subgroup-list.svelte-2bul55{margin-top:16px}.empty-state.svelte-2bul55{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-2bul55 h2:where(.svelte-2bul55){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-2bul55 p:where(.svelte-2bul55){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-2bul55 pre:where(.svelte-2bul55){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-2bul55 code:where(.svelte-2bul55){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-2bul55 pre:where(.svelte-2bul55) code:where(.svelte-2bul55){background:none;padding:0}.system-page.svelte-nv5os4{padding:20px 24px;overflow-y:auto;flex:1;min-height:0}.plot-grid.svelte-nv5os4{display:flex;flex-wrap:wrap;gap:12px}.empty-state.svelte-nv5os4{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-nv5os4 h2:where(.svelte-nv5os4){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-nv5os4 p:where(.svelte-nv5os4){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-nv5os4 pre:where(.svelte-nv5os4){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-nv5os4 ul:where(.svelte-nv5os4){list-style:disc;padding-left:20px;margin:4px 0 0}.empty-state.svelte-nv5os4 li:where(.svelte-nv5os4){margin:4px 0;color:var(--body-text-color, #1f2937)}.empty-state.svelte-nv5os4 code:where(.svelte-nv5os4){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-nv5os4 pre:where(.svelte-nv5os4) code:where(.svelte-nv5os4){background:none;padding:0}.table-container.svelte-1cp60rw{display:flex;flex-direction:column;gap:var(--size-2, 8px);position:relative}.header-row.svelte-1cp60rw{display:flex;justify-content:flex-end;align-items:center;min-height:var(--size-6, 24px);width:100%}.header-row.svelte-1cp60rw .label:where(.svelte-1cp60rw){flex:1;margin:0;color:var(--block-label-text-color, var(--neutral-500, #6b7280));font-size:var(--block-label-text-size, 12px);line-height:var(--line-sm, 1.4)}.table-wrap.svelte-1cp60rw{position:relative;overflow:auto;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--table-radius, var(--radius-lg, 8px))}table.svelte-1cp60rw{width:100%;table-layout:auto;color:var(--body-text-color, #1f2937);font-size:var(--input-text-size, 14px);line-height:var(--line-sm, 1.4);border-spacing:0;border-collapse:separate}thead.svelte-1cp60rw{position:sticky;top:0;z-index:5;box-shadow:var(--shadow-drop, rgba(0,0,0,.05) 0px 1px 2px 0px)}th.svelte-1cp60rw{padding:0;background:var(--table-even-background-fill, white);border-right-width:0px;border-left-width:1px;border-bottom-width:1px;border-style:solid;border-color:var(--border-color-primary, #e5e7eb);text-align:left;cursor:pointer;-webkit-user-select:none;user-select:none}th.first.svelte-1cp60rw{border-left-width:0;border-top-left-radius:var(--table-radius, var(--radius-lg, 8px))}th.last.svelte-1cp60rw{border-top-right-radius:var(--table-radius, var(--radius-lg, 8px))}.th-inner.svelte-1cp60rw{padding:var(--size-2, 8px);display:flex;align-items:center;gap:4px;font-weight:600;font-size:var(--text-sm, 12px);white-space:nowrap}.sort-arrow.svelte-1cp60rw{font-size:10px;color:var(--body-text-color-subdued, #9ca3af);visibility:hidden}.sort-arrow.visible.svelte-1cp60rw{visibility:visible}td.svelte-1cp60rw{padding:var(--size-2, 8px);border-right-width:0px;border-left-width:1px;border-bottom-width:1px;border-style:solid;border-color:var(--border-color-primary, #e5e7eb);font-size:var(--text-sm, 12px)}td.first.svelte-1cp60rw{border-left-width:0}tr.svelte-1cp60rw{background:var(--table-even-background-fill, white);border-bottom:1px solid var(--border-color-primary, #e5e7eb);text-align:left}tr.row-odd.svelte-1cp60rw{background:var(--table-odd-background-fill, var(--neutral-50, #f9fafb))}tr.selected.svelte-1cp60rw{background:var(--color-accent-soft, var(--primary-50, #fff7ed))}tr.svelte-1cp60rw:last-child td.first:where(.svelte-1cp60rw){border-bottom-left-radius:var(--table-radius, var(--radius-lg, 8px))}tr.svelte-1cp60rw:last-child td.last:where(.svelte-1cp60rw){border-bottom-right-radius:var(--table-radius, var(--radius-lg, 8px))}.check-col.svelte-1cp60rw{width:40px;text-align:center;padding:var(--size-2, 8px);border-left-width:0}.check-col.svelte-1cp60rw input[type=checkbox]:where(.svelte-1cp60rw){-moz-appearance:none;appearance:none;-webkit-appearance:none;width:16px;height:16px;border:1px solid var(--checkbox-border-color, #d1d5db);border-radius:var(--checkbox-border-radius, 4px);background-color:var(--checkbox-background-color, white);cursor:pointer;flex-shrink:0;transition:background-color .15s,border-color .15s}.check-col.svelte-1cp60rw input[type=checkbox]:where(.svelte-1cp60rw):checked{background-image:var(--checkbox-check);background-color:var(--checkbox-background-color-selected, #2563eb);border-color:var(--checkbox-border-color-selected, #2563eb)}.media-page.svelte-outb32{padding:20px 24px;overflow-y:auto;flex:1}.section-title.svelte-outb32{font-size:var(--text-lg, 16px);font-weight:600;color:var(--body-text-color, #1f2937);margin:16px 0 8px}.gallery.svelte-outb32{display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:12px}.gallery-item.svelte-outb32{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);overflow:hidden;background:var(--background-fill-secondary, #f9fafb)}.gallery-item.svelte-outb32 img:where(.svelte-outb32),.gallery-item.svelte-outb32 video:where(.svelte-outb32){width:100%;display:block}.caption.svelte-outb32{padding:4px 8px;font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #9ca3af)}.step-label.svelte-outb32{padding:4px 8px;font-size:var(--text-xs, 10px);color:var(--body-text-color-subdued, #9ca3af)}.audio-list.svelte-outb32{display:flex;flex-direction:column;gap:8px}.audio-item.svelte-outb32{display:flex;align-items:center;gap:12px;padding:8px;border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px)}.audio-label.svelte-outb32{font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #9ca3af);min-width:120px}.table-section.svelte-outb32{margin-bottom:16px}.empty-state.svelte-outb32{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-outb32 h2:where(.svelte-outb32){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-outb32 p:where(.svelte-outb32){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-outb32 pre:where(.svelte-outb32){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-outb32 code:where(.svelte-outb32){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-outb32 pre:where(.svelte-outb32) code:where(.svelte-outb32){background:none;padding:0}.reports-page.svelte-iufsej{padding:20px 24px;overflow-y:auto;flex:1}.controls.svelte-iufsej{display:flex;gap:16px;margin-bottom:16px;flex-wrap:wrap;align-items:flex-end}.control.svelte-iufsej{min-width:200px}.filter-pills.svelte-iufsej{display:flex;gap:4px}.pill.svelte-iufsej{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-xxl, 22px);padding:4px 12px;font-size:var(--text-sm, 12px);background:var(--background-fill-secondary, #f9fafb);color:var(--body-text-color-subdued, #6b7280);cursor:pointer;transition:background-color .15s,color .15s}.pill.svelte-iufsej:hover{background:var(--neutral-100, #f3f4f6)}.pill.active.svelte-iufsej{background:var(--color-accent, #f97316);color:#fff;border-color:var(--color-accent, #f97316)}.empty-state.svelte-iufsej{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-iufsej h2:where(.svelte-iufsej){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-iufsej p:where(.svelte-iufsej){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-iufsej pre:where(.svelte-iufsej){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-iufsej code:where(.svelte-iufsej){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-iufsej pre:where(.svelte-iufsej) code:where(.svelte-iufsej){background:none;padding:0}.alerts-table.svelte-iufsej{width:100%;border-collapse:collapse;font-size:var(--text-md, 14px)}.alerts-table.svelte-iufsej th:where(.svelte-iufsej){text-align:left;padding:8px 12px;border-bottom:2px solid var(--border-color-primary, #e5e7eb);color:var(--body-text-color-subdued, #6b7280);font-weight:600;font-size:var(--text-sm, 12px);text-transform:uppercase;letter-spacing:.05em}.alerts-table.svelte-iufsej td:where(.svelte-iufsej){padding:8px 12px;border-bottom:1px solid var(--border-color-primary, #e5e7eb);color:var(--body-text-color, #1f2937)}.alerts-table.svelte-iufsej tbody:where(.svelte-iufsej) tr:where(.svelte-iufsej):nth-child(odd){background:var(--table-odd-background-fill, var(--background-fill-primary, white))}.alerts-table.svelte-iufsej tbody:where(.svelte-iufsej) tr:where(.svelte-iufsej):nth-child(2n){background:var(--table-even-background-fill, var(--background-fill-secondary, #f9fafb))}.alerts-table.svelte-iufsej tr:where(.svelte-iufsej):hover{background:var(--background-fill-secondary, #f3f4f6)}.section-title.svelte-iufsej{font-size:16px;font-weight:700;margin:0 0 12px;color:var(--body-text-color, #1f2937)}.reports-section.svelte-iufsej,.alerts-section.svelte-iufsej{margin-bottom:32px}.report-card.svelte-iufsej{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);padding:16px 20px;margin-bottom:12px;background:var(--background-fill-primary, white)}.report-meta.svelte-iufsej{font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #6b7280);margin-bottom:8px}.report-content.svelte-iufsej{font-size:var(--text-md, 14px);color:var(--body-text-color, #1f2937);line-height:1.6}.report-content.svelte-iufsej h2{font-size:18px;font-weight:700;margin:0 0 8px}.report-content.svelte-iufsej h3{font-size:16px;font-weight:600;margin:12px 0 6px}.report-content.svelte-iufsej h4{font-size:14px;font-weight:600;margin:10px 0 4px}.report-content.svelte-iufsej code{background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.report-content.svelte-iufsej ul{margin:4px 0;padding-left:20px}.report-content.svelte-iufsej li{margin:2px 0}.report-content.svelte-iufsej p{margin:6px 0}.filter-empty.svelte-iufsej{color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-md, 14px)}.runs-page.svelte-1yb6d54{padding:20px 24px;overflow-y:auto;flex:1}.empty-state.svelte-1yb6d54{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-1yb6d54 h2:where(.svelte-1yb6d54){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-1yb6d54 p:where(.svelte-1yb6d54){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-1yb6d54 pre:where(.svelte-1yb6d54){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-1yb6d54 code:where(.svelte-1yb6d54){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-1yb6d54 pre:where(.svelte-1yb6d54) code:where(.svelte-1yb6d54){background:none;padding:0}.runs-table.svelte-1yb6d54{width:100%;border-collapse:collapse;font-size:var(--text-md, 14px)}.runs-table.svelte-1yb6d54 th:where(.svelte-1yb6d54){text-align:left;padding:8px 12px;border-bottom:2px solid var(--border-color-primary, #e5e7eb);color:var(--body-text-color-subdued, #6b7280);font-weight:600;font-size:var(--text-sm, 12px);text-transform:uppercase;letter-spacing:.05em}.runs-table.svelte-1yb6d54 td:where(.svelte-1yb6d54){padding:8px 12px;border-bottom:1px solid var(--border-color-primary, #e5e7eb);color:var(--body-text-color, #1f2937)}.runs-table.svelte-1yb6d54 tbody:where(.svelte-1yb6d54) tr:where(.svelte-1yb6d54):nth-child(odd){background:var(--table-odd-background-fill, var(--background-fill-primary, white))}.runs-table.svelte-1yb6d54 tbody:where(.svelte-1yb6d54) tr:where(.svelte-1yb6d54):nth-child(2n){background:var(--table-even-background-fill, var(--background-fill-secondary, #f9fafb))}.runs-table.svelte-1yb6d54 tr:where(.svelte-1yb6d54):hover{background:var(--background-fill-secondary, #f3f4f6)}.run-name-cell.svelte-1yb6d54{font-weight:500}.run-name-with-dot.svelte-1yb6d54{display:inline-flex;align-items:center;gap:8px;max-width:100%}.run-dot.svelte-1yb6d54{width:10px;height:10px;border-radius:50%;flex-shrink:0}.link-btn.svelte-1yb6d54{background:none;border:none;color:var(--color-accent, #f97316);cursor:pointer;font:inherit;font-weight:500;padding:0;text-align:left}.link-btn.svelte-1yb6d54:hover{text-decoration:underline}.rename-input.svelte-1yb6d54{font:inherit;padding:2px 6px;border:1px solid var(--color-accent, #f97316);border-radius:var(--radius-sm, 4px);outline:none;width:100%}.actions-cell.svelte-1yb6d54{display:flex;gap:4px}.action-btn.svelte-1yb6d54{background:none;border:1px solid transparent;color:var(--body-text-color-subdued, #6b7280);cursor:pointer;padding:4px;border-radius:var(--radius-sm, 4px);display:flex;align-items:center}.action-btn.svelte-1yb6d54:hover{background:var(--background-fill-secondary, #f9fafb);border-color:var(--border-color-primary, #e5e7eb);color:var(--body-text-color, #1f2937)}.delete-btn.svelte-1yb6d54:hover{color:#dc2626;border-color:#fecaca;background:#fef2f2}.action-btn.svelte-1yb6d54:disabled{opacity:.45;cursor:not-allowed;pointer-events:none}.run-detail-page.svelte-1bpgsx2{padding:20px 24px;overflow-y:auto;flex:1}.detail-card.svelte-1bpgsx2{background:var(--background-fill-primary, white);border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);padding:24px;max-width:800px}.detail-card.svelte-1bpgsx2 h2:where(.svelte-1bpgsx2){color:var(--body-text-color, #1f2937);margin:0 0 16px;font-size:var(--text-xl, 22px)}.detail-card.svelte-1bpgsx2 h3:where(.svelte-1bpgsx2){color:var(--body-text-color, #1f2937);margin:20px 0 8px;font-size:var(--text-lg, 16px)}.detail-grid.svelte-1bpgsx2{display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:12px}.detail-item.svelte-1bpgsx2{display:flex;flex-direction:column;gap:2px}.detail-label.svelte-1bpgsx2{font-size:var(--text-xs, 10px);font-weight:600;color:var(--body-text-color-subdued, #9ca3af);text-transform:uppercase}.detail-value.svelte-1bpgsx2{font-size:var(--text-md, 14px);color:var(--body-text-color, #1f2937)}.config-block.svelte-1bpgsx2{background:var(--background-fill-secondary, #f9fafb);padding:12px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:var(--text-sm, 12px);color:var(--body-text-color, #1f2937);overflow-x:auto}.empty-state.svelte-1bpgsx2{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-1bpgsx2 h2:where(.svelte-1bpgsx2){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-1bpgsx2 p:where(.svelte-1bpgsx2){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-1bpgsx2 pre:where(.svelte-1bpgsx2){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-1bpgsx2 code:where(.svelte-1bpgsx2){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-1bpgsx2 pre:where(.svelte-1bpgsx2) code:where(.svelte-1bpgsx2){background:none;padding:0}.files-page.svelte-1xvfk9n{padding:20px 24px;overflow-y:auto;flex:1}.page-title.svelte-1xvfk9n{color:var(--body-text-color, #1f2937);font-size:16px;font-weight:700;margin:0 0 4px}.page-subtitle.svelte-1xvfk9n{color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-sm, 12px);margin:0 0 16px}.file-list.svelte-1xvfk9n{display:flex;flex-direction:column;gap:4px}.file-item.svelte-1xvfk9n{border:1px solid var(--border-color-primary, #e5e7eb);border-radius:var(--radius-lg, 8px);background:var(--background-fill-primary, white);overflow:hidden}.file-item.expanded.svelte-1xvfk9n{border-color:var(--color-accent, #f97316)}.file-row.svelte-1xvfk9n{display:flex;align-items:center;justify-content:space-between;padding:10px 14px;gap:12px}.file-name.svelte-1xvfk9n{display:flex;align-items:center;gap:8px;background:none;border:none;padding:0;font-size:var(--text-md, 14px);color:var(--body-text-color, #1f2937);cursor:pointer;text-align:left}.file-name.svelte-1xvfk9n:hover{color:var(--color-accent, #f97316)}.file-icon.svelte-1xvfk9n{font-size:14px;flex-shrink:0}.file-actions.svelte-1xvfk9n{display:flex;align-items:center;gap:12px;flex-shrink:0}.file-size.svelte-1xvfk9n{font-size:var(--text-sm, 12px);color:var(--body-text-color-subdued, #6b7280);white-space:nowrap}.download-btn.svelte-1xvfk9n{display:flex;align-items:center;justify-content:center;width:28px;height:28px;border-radius:var(--radius-md, 6px);color:var(--body-text-color-subdued, #6b7280);transition:background-color .15s,color .15s}.download-btn.svelte-1xvfk9n:hover{background:var(--background-fill-secondary, #f3f4f6);color:var(--body-text-color, #1f2937)}.file-preview.svelte-1xvfk9n{border-top:1px solid var(--border-color-primary, #e5e7eb);padding:12px 14px;background:var(--background-fill-secondary, #f9fafb)}.preview-code.svelte-1xvfk9n{margin:0;font-size:12px;line-height:1.5;max-height:400px;overflow:auto;white-space:pre-wrap;word-break:break-all;color:var(--body-text-color, #1f2937)}.preview-loading.svelte-1xvfk9n,.preview-unavailable.svelte-1xvfk9n{color:var(--body-text-color-subdued, #6b7280);font-size:var(--text-sm, 12px);padding:8px 0}.preview-unavailable.svelte-1xvfk9n a:where(.svelte-1xvfk9n){color:var(--color-accent, #f97316);text-decoration:none}.preview-unavailable.svelte-1xvfk9n a:where(.svelte-1xvfk9n):hover{text-decoration:underline}.empty-state.svelte-1xvfk9n{max-width:640px;padding:40px 24px;color:var(--body-text-color, #1f2937)}.empty-state.svelte-1xvfk9n h2:where(.svelte-1xvfk9n){margin:0 0 8px;font-size:20px;font-weight:700}.empty-state.svelte-1xvfk9n p:where(.svelte-1xvfk9n){margin:12px 0 8px;color:var(--body-text-color-subdued, #6b7280)}.empty-state.svelte-1xvfk9n pre:where(.svelte-1xvfk9n){background:var(--background-fill-secondary, #f9fafb);padding:16px;border-radius:var(--radius-lg, 8px);border:1px solid var(--border-color-primary, #e5e7eb);font-size:13px;overflow-x:auto}.empty-state.svelte-1xvfk9n code:where(.svelte-1xvfk9n){background:var(--background-fill-secondary, #f0f0f0);padding:1px 5px;border-radius:var(--radius-sm, 4px);font-size:13px}.empty-state.svelte-1xvfk9n pre:where(.svelte-1xvfk9n) code:where(.svelte-1xvfk9n){background:none;padding:0}*{margin:0;padding:0;box-sizing:border-box}body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,sans-serif;background:var(--background-fill-primary, #fff);color:var(--body-text-color, #1f2937);font-size:var(--text-md, 14px);-webkit-font-smoothing:antialiased}.app.svelte-1n46o8q{display:flex;height:100vh;overflow:hidden}.main.svelte-1n46o8q{flex:1;display:flex;flex-direction:column;overflow:hidden;min-width:0}.page-content.svelte-1n46o8q{flex:1;overflow:hidden;display:flex;background:var(--bg-primary)}
|
trackio/frontend/dist/assets/index-UYQ1xITV.js
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
trackio/frontend/dist/index.html
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="UTF-8" />
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
| 6 |
+
<title>Trackio Dashboard</title>
|
| 7 |
+
<link rel="icon" type="image/png" href="/static/trackio/trackio_logo_light.png" />
|
| 8 |
+
<script type="module" crossorigin src="/assets/index-UYQ1xITV.js"></script>
|
| 9 |
+
<link rel="stylesheet" crossorigin href="/assets/index-Cs1UkfOV.css">
|
| 10 |
+
</head>
|
| 11 |
+
<body>
|
| 12 |
+
<div id="app"></div>
|
| 13 |
+
</body>
|
| 14 |
+
</html>
|
trackio/frontend/eslint.config.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import js from "@eslint/js";
|
| 2 |
+
import svelte from "eslint-plugin-svelte";
|
| 3 |
+
import svelteParser from "svelte-eslint-parser";
|
| 4 |
+
import globals from "globals";
|
| 5 |
+
|
| 6 |
+
export default [
|
| 7 |
+
{ ignores: ["dist/**", "node_modules/**"] },
|
| 8 |
+
{
|
| 9 |
+
files: ["**/*.js"],
|
| 10 |
+
languageOptions: {
|
| 11 |
+
globals: {
|
| 12 |
+
...globals.browser,
|
| 13 |
+
...globals.es2021,
|
| 14 |
+
$state: "readonly",
|
| 15 |
+
$derived: "readonly",
|
| 16 |
+
$effect: "readonly",
|
| 17 |
+
$props: "readonly",
|
| 18 |
+
$bindable: "readonly",
|
| 19 |
+
$inspect: "readonly",
|
| 20 |
+
},
|
| 21 |
+
},
|
| 22 |
+
rules: {
|
| 23 |
+
...js.configs.recommended.rules,
|
| 24 |
+
"no-unused-vars": ["error", { argsIgnorePattern: "^_" }],
|
| 25 |
+
"no-empty": "off",
|
| 26 |
+
},
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
files: ["**/*.svelte"],
|
| 30 |
+
languageOptions: {
|
| 31 |
+
parser: svelteParser,
|
| 32 |
+
globals: { ...globals.browser, ...globals.es2021 },
|
| 33 |
+
},
|
| 34 |
+
plugins: { svelte },
|
| 35 |
+
rules: {
|
| 36 |
+
...js.configs.recommended.rules,
|
| 37 |
+
...svelte.configs.recommended.rules,
|
| 38 |
+
"no-unused-vars": ["error", { argsIgnorePattern: "^_", varsIgnorePattern: "^\\$" }],
|
| 39 |
+
"no-empty": "off",
|
| 40 |
+
},
|
| 41 |
+
},
|
| 42 |
+
];
|
trackio/frontend/index.html
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="UTF-8" />
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
| 6 |
+
<title>Trackio Dashboard</title>
|
| 7 |
+
<link rel="icon" type="image/png" href="/static/trackio/trackio_logo_light.png" />
|
| 8 |
+
</head>
|
| 9 |
+
<body>
|
| 10 |
+
<div id="app"></div>
|
| 11 |
+
<script type="module" src="/src/main.js"></script>
|
| 12 |
+
</body>
|
| 13 |
+
</html>
|
trackio/frontend_server.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Serves the built Svelte frontend alongside the Gradio API."""
|
| 2 |
+
|
| 3 |
+
import logging
|
| 4 |
+
import re
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
from starlette.responses import HTMLResponse
|
| 8 |
+
from starlette.routing import Mount, Route
|
| 9 |
+
from starlette.staticfiles import StaticFiles
|
| 10 |
+
|
| 11 |
+
FRONTEND_DIR = Path(__file__).parent / "frontend" / "dist"
|
| 12 |
+
ASSETS_DIR = Path(__file__).parent / "assets"
|
| 13 |
+
|
| 14 |
+
_logger = logging.getLogger(__name__)
|
| 15 |
+
|
| 16 |
+
_SPA_SEGMENTS = (
|
| 17 |
+
"metrics",
|
| 18 |
+
"system",
|
| 19 |
+
"media",
|
| 20 |
+
"reports",
|
| 21 |
+
"runs",
|
| 22 |
+
"run",
|
| 23 |
+
"files",
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def mount_frontend(app):
|
| 28 |
+
if not FRONTEND_DIR.exists():
|
| 29 |
+
_logger.warning(
|
| 30 |
+
"Trackio dashboard UI was not mounted: %s is missing. "
|
| 31 |
+
"Build the frontend with `npm ci && npm run build` in trackio/frontend.",
|
| 32 |
+
FRONTEND_DIR,
|
| 33 |
+
)
|
| 34 |
+
return
|
| 35 |
+
|
| 36 |
+
index_html_path = FRONTEND_DIR / "index.html"
|
| 37 |
+
if not index_html_path.exists():
|
| 38 |
+
_logger.warning(
|
| 39 |
+
"Trackio dashboard UI was not mounted: %s is missing.",
|
| 40 |
+
index_html_path,
|
| 41 |
+
)
|
| 42 |
+
return
|
| 43 |
+
|
| 44 |
+
index_html_content = index_html_path.read_text()
|
| 45 |
+
patched_html = re.sub(
|
| 46 |
+
r'/assets/(index-[^"]+)',
|
| 47 |
+
r"/assets/app/\1",
|
| 48 |
+
index_html_content,
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
async def serve_frontend(request):
|
| 52 |
+
return HTMLResponse(patched_html)
|
| 53 |
+
|
| 54 |
+
vite_assets = StaticFiles(directory=str(FRONTEND_DIR / "assets"))
|
| 55 |
+
static_assets = StaticFiles(directory=str(ASSETS_DIR))
|
| 56 |
+
|
| 57 |
+
app.routes.insert(0, Mount("/static/trackio", app=static_assets))
|
| 58 |
+
app.routes.insert(0, Mount("/assets/app", app=vite_assets))
|
| 59 |
+
|
| 60 |
+
for seg in reversed(_SPA_SEGMENTS):
|
| 61 |
+
app.routes.insert(0, Route(f"/{seg}/", serve_frontend, methods=["GET"]))
|
| 62 |
+
app.routes.insert(0, Route(f"/{seg}", serve_frontend, methods=["GET"]))
|
| 63 |
+
app.routes.insert(0, Route("/", serve_frontend, methods=["GET"]))
|
trackio/gpu.py
ADDED
|
@@ -0,0 +1,357 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import threading
|
| 3 |
+
import warnings
|
| 4 |
+
from typing import TYPE_CHECKING, Any
|
| 5 |
+
|
| 6 |
+
if TYPE_CHECKING:
|
| 7 |
+
from trackio.run import Run
|
| 8 |
+
|
| 9 |
+
pynvml: Any = None
|
| 10 |
+
PYNVML_AVAILABLE = False
|
| 11 |
+
_nvml_initialized = False
|
| 12 |
+
_nvml_lock = threading.Lock()
|
| 13 |
+
_energy_baseline: dict[int, float] = {}
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _ensure_pynvml():
|
| 17 |
+
global PYNVML_AVAILABLE, pynvml
|
| 18 |
+
if PYNVML_AVAILABLE:
|
| 19 |
+
return pynvml
|
| 20 |
+
try:
|
| 21 |
+
import pynvml as _pynvml
|
| 22 |
+
|
| 23 |
+
pynvml = _pynvml
|
| 24 |
+
PYNVML_AVAILABLE = True
|
| 25 |
+
return pynvml
|
| 26 |
+
except ImportError:
|
| 27 |
+
raise ImportError(
|
| 28 |
+
"nvidia-ml-py is required for GPU monitoring. "
|
| 29 |
+
"Install it with: pip install nvidia-ml-py"
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _init_nvml() -> bool:
|
| 34 |
+
global _nvml_initialized
|
| 35 |
+
with _nvml_lock:
|
| 36 |
+
if _nvml_initialized:
|
| 37 |
+
return True
|
| 38 |
+
try:
|
| 39 |
+
nvml = _ensure_pynvml()
|
| 40 |
+
nvml.nvmlInit()
|
| 41 |
+
_nvml_initialized = True
|
| 42 |
+
return True
|
| 43 |
+
except Exception:
|
| 44 |
+
return False
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def get_gpu_count() -> tuple[int, list[int]]:
|
| 48 |
+
"""
|
| 49 |
+
Get the number of GPUs visible to this process and their physical indices.
|
| 50 |
+
Respects CUDA_VISIBLE_DEVICES environment variable.
|
| 51 |
+
|
| 52 |
+
Returns:
|
| 53 |
+
Tuple of (count, physical_indices) where:
|
| 54 |
+
- count: Number of visible GPUs
|
| 55 |
+
- physical_indices: List mapping logical index to physical GPU index.
|
| 56 |
+
e.g., if CUDA_VISIBLE_DEVICES=2,3 returns (2, [2, 3])
|
| 57 |
+
meaning logical GPU 0 = physical GPU 2, logical GPU 1 = physical GPU 3
|
| 58 |
+
"""
|
| 59 |
+
if not _init_nvml():
|
| 60 |
+
return 0, []
|
| 61 |
+
|
| 62 |
+
cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES")
|
| 63 |
+
if cuda_visible is not None and cuda_visible.strip():
|
| 64 |
+
try:
|
| 65 |
+
indices = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()]
|
| 66 |
+
return len(indices), indices
|
| 67 |
+
except ValueError:
|
| 68 |
+
pass
|
| 69 |
+
|
| 70 |
+
try:
|
| 71 |
+
total = pynvml.nvmlDeviceGetCount()
|
| 72 |
+
return total, list(range(total))
|
| 73 |
+
except Exception:
|
| 74 |
+
return 0, []
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def gpu_available() -> bool:
|
| 78 |
+
"""
|
| 79 |
+
Check if GPU monitoring is available.
|
| 80 |
+
|
| 81 |
+
Returns True if nvidia-ml-py is installed and at least one NVIDIA GPU is detected.
|
| 82 |
+
This is used for auto-detection of GPU logging.
|
| 83 |
+
"""
|
| 84 |
+
try:
|
| 85 |
+
_ensure_pynvml()
|
| 86 |
+
count, _ = get_gpu_count()
|
| 87 |
+
return count > 0
|
| 88 |
+
except ImportError:
|
| 89 |
+
return False
|
| 90 |
+
except Exception:
|
| 91 |
+
return False
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def reset_energy_baseline():
|
| 95 |
+
"""Reset the energy baseline for all GPUs. Called when a new run starts."""
|
| 96 |
+
global _energy_baseline
|
| 97 |
+
_energy_baseline = {}
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def collect_gpu_metrics(device: int | None = None) -> dict:
|
| 101 |
+
"""
|
| 102 |
+
Collect GPU metrics for visible GPUs.
|
| 103 |
+
|
| 104 |
+
Args:
|
| 105 |
+
device: CUDA device index to collect metrics from. If None, collects
|
| 106 |
+
from all GPUs visible to this process (respects CUDA_VISIBLE_DEVICES).
|
| 107 |
+
The device index is the logical CUDA index (0, 1, 2...), not the
|
| 108 |
+
physical GPU index.
|
| 109 |
+
|
| 110 |
+
Returns:
|
| 111 |
+
Dictionary of GPU metrics. Keys use logical device indices (gpu/0/, gpu/1/, etc.)
|
| 112 |
+
which correspond to CUDA device indices, not physical GPU indices.
|
| 113 |
+
"""
|
| 114 |
+
if not _init_nvml():
|
| 115 |
+
return {}
|
| 116 |
+
|
| 117 |
+
gpu_count, visible_gpus = get_gpu_count()
|
| 118 |
+
if gpu_count == 0:
|
| 119 |
+
return {}
|
| 120 |
+
|
| 121 |
+
if device is not None:
|
| 122 |
+
if device < 0 or device >= gpu_count:
|
| 123 |
+
return {}
|
| 124 |
+
gpu_indices = [(device, visible_gpus[device])]
|
| 125 |
+
else:
|
| 126 |
+
gpu_indices = list(enumerate(visible_gpus))
|
| 127 |
+
|
| 128 |
+
metrics = {}
|
| 129 |
+
total_util = 0.0
|
| 130 |
+
total_mem_used_gib = 0.0
|
| 131 |
+
total_power = 0.0
|
| 132 |
+
max_temp = 0.0
|
| 133 |
+
valid_util_count = 0
|
| 134 |
+
|
| 135 |
+
for logical_idx, physical_idx in gpu_indices:
|
| 136 |
+
prefix = f"gpu/{logical_idx}"
|
| 137 |
+
try:
|
| 138 |
+
handle = pynvml.nvmlDeviceGetHandleByIndex(physical_idx)
|
| 139 |
+
|
| 140 |
+
try:
|
| 141 |
+
util = pynvml.nvmlDeviceGetUtilizationRates(handle)
|
| 142 |
+
metrics[f"{prefix}/utilization"] = util.gpu
|
| 143 |
+
metrics[f"{prefix}/memory_utilization"] = util.memory
|
| 144 |
+
total_util += util.gpu
|
| 145 |
+
valid_util_count += 1
|
| 146 |
+
except Exception:
|
| 147 |
+
pass
|
| 148 |
+
|
| 149 |
+
try:
|
| 150 |
+
mem = pynvml.nvmlDeviceGetMemoryInfo(handle)
|
| 151 |
+
mem_used_gib = mem.used / (1024**3)
|
| 152 |
+
mem_total_gib = mem.total / (1024**3)
|
| 153 |
+
metrics[f"{prefix}/allocated_memory"] = mem_used_gib
|
| 154 |
+
metrics[f"{prefix}/total_memory"] = mem_total_gib
|
| 155 |
+
if mem.total > 0:
|
| 156 |
+
metrics[f"{prefix}/memory_usage"] = mem.used / mem.total
|
| 157 |
+
total_mem_used_gib += mem_used_gib
|
| 158 |
+
except Exception:
|
| 159 |
+
pass
|
| 160 |
+
|
| 161 |
+
try:
|
| 162 |
+
power_mw = pynvml.nvmlDeviceGetPowerUsage(handle)
|
| 163 |
+
power_w = power_mw / 1000.0
|
| 164 |
+
metrics[f"{prefix}/power"] = power_w
|
| 165 |
+
total_power += power_w
|
| 166 |
+
except Exception:
|
| 167 |
+
pass
|
| 168 |
+
|
| 169 |
+
try:
|
| 170 |
+
power_limit_mw = pynvml.nvmlDeviceGetPowerManagementLimit(handle)
|
| 171 |
+
power_limit_w = power_limit_mw / 1000.0
|
| 172 |
+
metrics[f"{prefix}/power_limit"] = power_limit_w
|
| 173 |
+
if power_limit_w > 0 and f"{prefix}/power" in metrics:
|
| 174 |
+
metrics[f"{prefix}/power_percent"] = (
|
| 175 |
+
metrics[f"{prefix}/power"] / power_limit_w
|
| 176 |
+
) * 100
|
| 177 |
+
except Exception:
|
| 178 |
+
pass
|
| 179 |
+
|
| 180 |
+
try:
|
| 181 |
+
temp = pynvml.nvmlDeviceGetTemperature(
|
| 182 |
+
handle, pynvml.NVML_TEMPERATURE_GPU
|
| 183 |
+
)
|
| 184 |
+
metrics[f"{prefix}/temp"] = temp
|
| 185 |
+
max_temp = max(max_temp, temp)
|
| 186 |
+
except Exception:
|
| 187 |
+
pass
|
| 188 |
+
|
| 189 |
+
try:
|
| 190 |
+
sm_clock = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_SM)
|
| 191 |
+
metrics[f"{prefix}/sm_clock"] = sm_clock
|
| 192 |
+
except Exception:
|
| 193 |
+
pass
|
| 194 |
+
|
| 195 |
+
try:
|
| 196 |
+
mem_clock = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_MEM)
|
| 197 |
+
metrics[f"{prefix}/memory_clock"] = mem_clock
|
| 198 |
+
except Exception:
|
| 199 |
+
pass
|
| 200 |
+
|
| 201 |
+
try:
|
| 202 |
+
fan_speed = pynvml.nvmlDeviceGetFanSpeed(handle)
|
| 203 |
+
metrics[f"{prefix}/fan_speed"] = fan_speed
|
| 204 |
+
except Exception:
|
| 205 |
+
pass
|
| 206 |
+
|
| 207 |
+
try:
|
| 208 |
+
pstate = pynvml.nvmlDeviceGetPerformanceState(handle)
|
| 209 |
+
metrics[f"{prefix}/performance_state"] = pstate
|
| 210 |
+
except Exception:
|
| 211 |
+
pass
|
| 212 |
+
|
| 213 |
+
try:
|
| 214 |
+
energy_mj = pynvml.nvmlDeviceGetTotalEnergyConsumption(handle)
|
| 215 |
+
if logical_idx not in _energy_baseline:
|
| 216 |
+
_energy_baseline[logical_idx] = energy_mj
|
| 217 |
+
energy_consumed_mj = energy_mj - _energy_baseline[logical_idx]
|
| 218 |
+
metrics[f"{prefix}/energy_consumed"] = energy_consumed_mj / 1000.0
|
| 219 |
+
except Exception:
|
| 220 |
+
pass
|
| 221 |
+
|
| 222 |
+
try:
|
| 223 |
+
pcie_tx = pynvml.nvmlDeviceGetPcieThroughput(
|
| 224 |
+
handle, pynvml.NVML_PCIE_UTIL_TX_BYTES
|
| 225 |
+
)
|
| 226 |
+
pcie_rx = pynvml.nvmlDeviceGetPcieThroughput(
|
| 227 |
+
handle, pynvml.NVML_PCIE_UTIL_RX_BYTES
|
| 228 |
+
)
|
| 229 |
+
metrics[f"{prefix}/pcie_tx"] = pcie_tx / 1024.0
|
| 230 |
+
metrics[f"{prefix}/pcie_rx"] = pcie_rx / 1024.0
|
| 231 |
+
except Exception:
|
| 232 |
+
pass
|
| 233 |
+
|
| 234 |
+
try:
|
| 235 |
+
throttle = pynvml.nvmlDeviceGetCurrentClocksThrottleReasons(handle)
|
| 236 |
+
metrics[f"{prefix}/throttle_thermal"] = int(
|
| 237 |
+
bool(throttle & pynvml.nvmlClocksThrottleReasonSwThermalSlowdown)
|
| 238 |
+
)
|
| 239 |
+
metrics[f"{prefix}/throttle_power"] = int(
|
| 240 |
+
bool(throttle & pynvml.nvmlClocksThrottleReasonSwPowerCap)
|
| 241 |
+
)
|
| 242 |
+
metrics[f"{prefix}/throttle_hw_slowdown"] = int(
|
| 243 |
+
bool(throttle & pynvml.nvmlClocksThrottleReasonHwSlowdown)
|
| 244 |
+
)
|
| 245 |
+
metrics[f"{prefix}/throttle_apps"] = int(
|
| 246 |
+
bool(
|
| 247 |
+
throttle
|
| 248 |
+
& pynvml.nvmlClocksThrottleReasonApplicationsClocksSetting
|
| 249 |
+
)
|
| 250 |
+
)
|
| 251 |
+
except Exception:
|
| 252 |
+
pass
|
| 253 |
+
|
| 254 |
+
try:
|
| 255 |
+
ecc_corrected = pynvml.nvmlDeviceGetTotalEccErrors(
|
| 256 |
+
handle,
|
| 257 |
+
pynvml.NVML_MEMORY_ERROR_TYPE_CORRECTED,
|
| 258 |
+
pynvml.NVML_VOLATILE_ECC,
|
| 259 |
+
)
|
| 260 |
+
metrics[f"{prefix}/corrected_memory_errors"] = ecc_corrected
|
| 261 |
+
except Exception:
|
| 262 |
+
pass
|
| 263 |
+
|
| 264 |
+
try:
|
| 265 |
+
ecc_uncorrected = pynvml.nvmlDeviceGetTotalEccErrors(
|
| 266 |
+
handle,
|
| 267 |
+
pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED,
|
| 268 |
+
pynvml.NVML_VOLATILE_ECC,
|
| 269 |
+
)
|
| 270 |
+
metrics[f"{prefix}/uncorrected_memory_errors"] = ecc_uncorrected
|
| 271 |
+
except Exception:
|
| 272 |
+
pass
|
| 273 |
+
|
| 274 |
+
except Exception:
|
| 275 |
+
continue
|
| 276 |
+
|
| 277 |
+
if valid_util_count > 0:
|
| 278 |
+
metrics["gpu/mean_utilization"] = total_util / valid_util_count
|
| 279 |
+
if total_mem_used_gib > 0:
|
| 280 |
+
metrics["gpu/total_allocated_memory"] = total_mem_used_gib
|
| 281 |
+
if total_power > 0:
|
| 282 |
+
metrics["gpu/total_power"] = total_power
|
| 283 |
+
if max_temp > 0:
|
| 284 |
+
metrics["gpu/max_temp"] = max_temp
|
| 285 |
+
|
| 286 |
+
return metrics
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
class GpuMonitor:
|
| 290 |
+
def __init__(self, run: "Run", interval: float = 10.0):
|
| 291 |
+
self._run = run
|
| 292 |
+
self._interval = interval
|
| 293 |
+
self._stop_flag = threading.Event()
|
| 294 |
+
self._thread: "threading.Thread | None" = None
|
| 295 |
+
|
| 296 |
+
def start(self):
|
| 297 |
+
count, _ = get_gpu_count()
|
| 298 |
+
if count == 0:
|
| 299 |
+
warnings.warn(
|
| 300 |
+
"auto_log_gpu=True but no NVIDIA GPUs detected. GPU logging disabled."
|
| 301 |
+
)
|
| 302 |
+
return
|
| 303 |
+
|
| 304 |
+
reset_energy_baseline()
|
| 305 |
+
self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
|
| 306 |
+
self._thread.start()
|
| 307 |
+
|
| 308 |
+
def stop(self):
|
| 309 |
+
self._stop_flag.set()
|
| 310 |
+
if self._thread is not None:
|
| 311 |
+
self._thread.join(timeout=2.0)
|
| 312 |
+
|
| 313 |
+
def _monitor_loop(self):
|
| 314 |
+
while not self._stop_flag.is_set():
|
| 315 |
+
try:
|
| 316 |
+
metrics = collect_gpu_metrics()
|
| 317 |
+
if metrics:
|
| 318 |
+
self._run.log_system(metrics)
|
| 319 |
+
except Exception:
|
| 320 |
+
pass
|
| 321 |
+
|
| 322 |
+
self._stop_flag.wait(timeout=self._interval)
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def log_gpu(run: "Run | None" = None, device: int | None = None) -> dict:
|
| 326 |
+
"""
|
| 327 |
+
Log GPU metrics to the current or specified run as system metrics.
|
| 328 |
+
|
| 329 |
+
Args:
|
| 330 |
+
run: Optional Run instance. If None, uses current run from context.
|
| 331 |
+
device: CUDA device index to collect metrics from. If None, collects
|
| 332 |
+
from all GPUs visible to this process (respects CUDA_VISIBLE_DEVICES).
|
| 333 |
+
|
| 334 |
+
Returns:
|
| 335 |
+
dict: The GPU metrics that were logged.
|
| 336 |
+
|
| 337 |
+
Example:
|
| 338 |
+
```python
|
| 339 |
+
import trackio
|
| 340 |
+
|
| 341 |
+
run = trackio.init(project="my-project")
|
| 342 |
+
trackio.log({"loss": 0.5})
|
| 343 |
+
trackio.log_gpu() # logs all visible GPUs
|
| 344 |
+
trackio.log_gpu(device=0) # logs only CUDA device 0
|
| 345 |
+
```
|
| 346 |
+
"""
|
| 347 |
+
from trackio import context_vars
|
| 348 |
+
|
| 349 |
+
if run is None:
|
| 350 |
+
run = context_vars.current_run.get()
|
| 351 |
+
if run is None:
|
| 352 |
+
raise RuntimeError("Call trackio.init() before trackio.log_gpu().")
|
| 353 |
+
|
| 354 |
+
metrics = collect_gpu_metrics(device=device)
|
| 355 |
+
if metrics:
|
| 356 |
+
run.log_system(metrics)
|
| 357 |
+
return metrics
|
trackio/histogram.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Sequence
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class Histogram:
|
| 7 |
+
"""
|
| 8 |
+
Histogram data type for Trackio, compatible with wandb.Histogram.
|
| 9 |
+
|
| 10 |
+
Args:
|
| 11 |
+
sequence (`np.ndarray` or `Sequence[float]` or `Sequence[int]`, *optional*):
|
| 12 |
+
Sequence of values to create the histogram from.
|
| 13 |
+
np_histogram (`tuple`, *optional*):
|
| 14 |
+
Pre-computed NumPy histogram as a `(hist, bins)` tuple.
|
| 15 |
+
num_bins (`int`, *optional*, defaults to `64`):
|
| 16 |
+
Number of bins for the histogram (maximum `512`).
|
| 17 |
+
|
| 18 |
+
Example:
|
| 19 |
+
```python
|
| 20 |
+
import trackio
|
| 21 |
+
import numpy as np
|
| 22 |
+
|
| 23 |
+
# Create histogram from sequence
|
| 24 |
+
data = np.random.randn(1000)
|
| 25 |
+
trackio.log({"distribution": trackio.Histogram(data)})
|
| 26 |
+
|
| 27 |
+
# Create histogram from numpy histogram
|
| 28 |
+
hist, bins = np.histogram(data, bins=30)
|
| 29 |
+
trackio.log({"distribution": trackio.Histogram(np_histogram=(hist, bins))})
|
| 30 |
+
|
| 31 |
+
# Specify custom number of bins
|
| 32 |
+
trackio.log({"distribution": trackio.Histogram(data, num_bins=50)})
|
| 33 |
+
```
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
TYPE = "trackio.histogram"
|
| 37 |
+
|
| 38 |
+
def __init__(
|
| 39 |
+
self,
|
| 40 |
+
sequence: np.ndarray | Sequence[float] | Sequence[int] | None = None,
|
| 41 |
+
np_histogram: tuple | None = None,
|
| 42 |
+
num_bins: int = 64,
|
| 43 |
+
):
|
| 44 |
+
if sequence is None and np_histogram is None:
|
| 45 |
+
raise ValueError("Must provide either sequence or np_histogram")
|
| 46 |
+
|
| 47 |
+
if sequence is not None and np_histogram is not None:
|
| 48 |
+
raise ValueError("Cannot provide both sequence and np_histogram")
|
| 49 |
+
|
| 50 |
+
num_bins = min(num_bins, 512)
|
| 51 |
+
|
| 52 |
+
if np_histogram is not None:
|
| 53 |
+
self.histogram, self.bins = np_histogram
|
| 54 |
+
self.histogram = np.asarray(self.histogram)
|
| 55 |
+
self.bins = np.asarray(self.bins)
|
| 56 |
+
else:
|
| 57 |
+
data = np.asarray(sequence).flatten()
|
| 58 |
+
data = data[np.isfinite(data)]
|
| 59 |
+
if len(data) == 0:
|
| 60 |
+
self.histogram = np.array([])
|
| 61 |
+
self.bins = np.array([])
|
| 62 |
+
else:
|
| 63 |
+
self.histogram, self.bins = np.histogram(data, bins=num_bins)
|
| 64 |
+
|
| 65 |
+
def _to_dict(self) -> dict:
|
| 66 |
+
"""Convert histogram to dictionary for storage."""
|
| 67 |
+
return {
|
| 68 |
+
"_type": self.TYPE,
|
| 69 |
+
"bins": self.bins.tolist(),
|
| 70 |
+
"values": self.histogram.tolist(),
|
| 71 |
+
}
|
trackio/imports.py
ADDED
|
@@ -0,0 +1,290 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
import pandas as pd
|
| 5 |
+
|
| 6 |
+
from trackio import deploy, utils
|
| 7 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def import_csv(
|
| 11 |
+
csv_path: str | Path,
|
| 12 |
+
project: str,
|
| 13 |
+
name: str | None = None,
|
| 14 |
+
space_id: str | None = None,
|
| 15 |
+
dataset_id: str | None = None,
|
| 16 |
+
private: bool | None = None,
|
| 17 |
+
force: bool = False,
|
| 18 |
+
) -> None:
|
| 19 |
+
"""
|
| 20 |
+
Imports a CSV file into a Trackio project. The CSV file must contain a `"step"`
|
| 21 |
+
column, may optionally contain a `"timestamp"` column, and any other columns will be
|
| 22 |
+
treated as metrics. It should also include a header row with the column names.
|
| 23 |
+
|
| 24 |
+
TODO: call init() and return a Run object so that the user can continue to log metrics to it.
|
| 25 |
+
|
| 26 |
+
Args:
|
| 27 |
+
csv_path (`str` or `Path`):
|
| 28 |
+
The str or Path to the CSV file to import.
|
| 29 |
+
project (`str`):
|
| 30 |
+
The name of the project to import the CSV file into. Must not be an existing
|
| 31 |
+
project.
|
| 32 |
+
name (`str`, *optional*):
|
| 33 |
+
The name of the Run to import the CSV file into. If not provided, a default
|
| 34 |
+
name will be generated.
|
| 35 |
+
name (`str`, *optional*):
|
| 36 |
+
The name of the run (if not provided, a default name will be generated).
|
| 37 |
+
space_id (`str`, *optional*):
|
| 38 |
+
If provided, the project will be logged to a Hugging Face Space instead of a
|
| 39 |
+
local directory. Should be a complete Space name like `"username/reponame"`
|
| 40 |
+
or `"orgname/reponame"`, or just `"reponame"` in which case the Space will
|
| 41 |
+
be created in the currently-logged-in Hugging Face user's namespace. If the
|
| 42 |
+
Space does not exist, it will be created. If the Space already exists, the
|
| 43 |
+
project will be logged to it.
|
| 44 |
+
dataset_id (`str`, *optional*):
|
| 45 |
+
Deprecated. Use `bucket_id` instead.
|
| 46 |
+
private (`bool`, *optional*):
|
| 47 |
+
Whether to make the Space private. If None (default), the repo will be
|
| 48 |
+
public unless the organization's default is private. This value is ignored
|
| 49 |
+
if the repo already exists.
|
| 50 |
+
"""
|
| 51 |
+
if SQLiteStorage.get_runs(project):
|
| 52 |
+
raise ValueError(
|
| 53 |
+
f"Project '{project}' already exists. Cannot import CSV into existing project."
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
csv_path = Path(csv_path)
|
| 57 |
+
if not csv_path.exists():
|
| 58 |
+
raise FileNotFoundError(f"CSV file not found: {csv_path}")
|
| 59 |
+
|
| 60 |
+
df = pd.read_csv(csv_path)
|
| 61 |
+
if df.empty:
|
| 62 |
+
raise ValueError("CSV file is empty")
|
| 63 |
+
|
| 64 |
+
column_mapping = utils.simplify_column_names(df.columns.tolist())
|
| 65 |
+
df = df.rename(columns=column_mapping)
|
| 66 |
+
|
| 67 |
+
step_column = None
|
| 68 |
+
for col in df.columns:
|
| 69 |
+
if col.lower() == "step":
|
| 70 |
+
step_column = col
|
| 71 |
+
break
|
| 72 |
+
|
| 73 |
+
if step_column is None:
|
| 74 |
+
raise ValueError("CSV file must contain a 'step' or 'Step' column")
|
| 75 |
+
|
| 76 |
+
if name is None:
|
| 77 |
+
name = csv_path.stem
|
| 78 |
+
|
| 79 |
+
metrics_list = []
|
| 80 |
+
steps = []
|
| 81 |
+
timestamps = []
|
| 82 |
+
|
| 83 |
+
numeric_columns = []
|
| 84 |
+
for column in df.columns:
|
| 85 |
+
if column == step_column:
|
| 86 |
+
continue
|
| 87 |
+
if column == "timestamp":
|
| 88 |
+
continue
|
| 89 |
+
|
| 90 |
+
try:
|
| 91 |
+
pd.to_numeric(df[column], errors="raise")
|
| 92 |
+
numeric_columns.append(column)
|
| 93 |
+
except (ValueError, TypeError):
|
| 94 |
+
continue
|
| 95 |
+
|
| 96 |
+
for _, row in df.iterrows():
|
| 97 |
+
metrics = {}
|
| 98 |
+
for column in numeric_columns:
|
| 99 |
+
value = row[column]
|
| 100 |
+
if bool(pd.notna(value)):
|
| 101 |
+
metrics[column] = float(value)
|
| 102 |
+
|
| 103 |
+
if metrics:
|
| 104 |
+
metrics_list.append(metrics)
|
| 105 |
+
steps.append(int(row[step_column]))
|
| 106 |
+
|
| 107 |
+
if "timestamp" in df.columns and bool(pd.notna(row["timestamp"])):
|
| 108 |
+
timestamps.append(str(row["timestamp"]))
|
| 109 |
+
else:
|
| 110 |
+
timestamps.append("")
|
| 111 |
+
|
| 112 |
+
if metrics_list:
|
| 113 |
+
SQLiteStorage.bulk_log(
|
| 114 |
+
project=project,
|
| 115 |
+
run=name,
|
| 116 |
+
metrics_list=metrics_list,
|
| 117 |
+
steps=steps,
|
| 118 |
+
timestamps=timestamps,
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
print(
|
| 122 |
+
f"* Imported {len(metrics_list)} rows from {csv_path} into project '{project}' as run '{name}'"
|
| 123 |
+
)
|
| 124 |
+
print(f"* Metrics found: {', '.join(metrics_list[0].keys())}")
|
| 125 |
+
|
| 126 |
+
space_id, dataset_id, _ = utils.preprocess_space_and_dataset_ids(
|
| 127 |
+
space_id, dataset_id
|
| 128 |
+
)
|
| 129 |
+
if dataset_id is not None:
|
| 130 |
+
os.environ["TRACKIO_DATASET_ID"] = dataset_id
|
| 131 |
+
print(f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}")
|
| 132 |
+
|
| 133 |
+
if space_id is None:
|
| 134 |
+
utils.print_dashboard_instructions(project)
|
| 135 |
+
else:
|
| 136 |
+
deploy.create_space_if_not_exists(
|
| 137 |
+
space_id=space_id, dataset_id=dataset_id, private=private
|
| 138 |
+
)
|
| 139 |
+
deploy.wait_until_space_exists(space_id=space_id)
|
| 140 |
+
deploy.upload_db_to_space(project=project, space_id=space_id, force=force)
|
| 141 |
+
print(
|
| 142 |
+
f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}"
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def import_tf_events(
|
| 147 |
+
log_dir: str | Path,
|
| 148 |
+
project: str,
|
| 149 |
+
name: str | None = None,
|
| 150 |
+
space_id: str | None = None,
|
| 151 |
+
dataset_id: str | None = None,
|
| 152 |
+
private: bool | None = None,
|
| 153 |
+
force: bool = False,
|
| 154 |
+
) -> None:
|
| 155 |
+
"""
|
| 156 |
+
Imports TensorFlow Events files from a directory into a Trackio project. Each
|
| 157 |
+
subdirectory in the log directory will be imported as a separate run.
|
| 158 |
+
|
| 159 |
+
Args:
|
| 160 |
+
log_dir (`str` or `Path`):
|
| 161 |
+
The str or Path to the directory containing TensorFlow Events files.
|
| 162 |
+
project (`str`):
|
| 163 |
+
The name of the project to import the TensorFlow Events files into. Must not
|
| 164 |
+
be an existing project.
|
| 165 |
+
name (`str`, *optional*):
|
| 166 |
+
The name prefix for runs (if not provided, will use directory names). Each
|
| 167 |
+
subdirectory will create a separate run.
|
| 168 |
+
space_id (`str`, *optional*):
|
| 169 |
+
If provided, the project will be logged to a Hugging Face Space instead of a
|
| 170 |
+
local directory. Should be a complete Space name like `"username/reponame"`
|
| 171 |
+
or `"orgname/reponame"`, or just `"reponame"` in which case the Space will
|
| 172 |
+
be created in the currently-logged-in Hugging Face user's namespace. If the
|
| 173 |
+
Space does not exist, it will be created. If the Space already exists, the
|
| 174 |
+
project will be logged to it.
|
| 175 |
+
dataset_id (`str`, *optional*):
|
| 176 |
+
Deprecated. Use `bucket_id` instead.
|
| 177 |
+
private (`bool`, *optional*):
|
| 178 |
+
Whether to make the Space private. If None (default), the repo will be
|
| 179 |
+
public unless the organization's default is private. This value is ignored
|
| 180 |
+
if the repo already exists.
|
| 181 |
+
"""
|
| 182 |
+
try:
|
| 183 |
+
from tbparse import SummaryReader
|
| 184 |
+
except ImportError:
|
| 185 |
+
raise ImportError(
|
| 186 |
+
"The `tbparse` package is not installed but is required for `import_tf_events`. Please install trackio with the `tensorboard` extra: `pip install trackio[tensorboard]`."
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
if SQLiteStorage.get_runs(project):
|
| 190 |
+
raise ValueError(
|
| 191 |
+
f"Project '{project}' already exists. Cannot import TF events into existing project."
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
path = Path(log_dir)
|
| 195 |
+
if not path.exists():
|
| 196 |
+
raise FileNotFoundError(f"TF events directory not found: {path}")
|
| 197 |
+
|
| 198 |
+
# Use tbparse to read all tfevents files in the directory structure
|
| 199 |
+
reader = SummaryReader(str(path), extra_columns={"dir_name"})
|
| 200 |
+
df = reader.scalars
|
| 201 |
+
|
| 202 |
+
if df.empty:
|
| 203 |
+
raise ValueError(f"No TensorFlow events data found in {path}")
|
| 204 |
+
|
| 205 |
+
total_imported = 0
|
| 206 |
+
imported_runs = []
|
| 207 |
+
|
| 208 |
+
# Group by dir_name to create separate runs
|
| 209 |
+
for dir_name, group_df in df.groupby("dir_name"):
|
| 210 |
+
try:
|
| 211 |
+
# Determine run name based on directory name
|
| 212 |
+
if dir_name == "":
|
| 213 |
+
run_name = "main" # For files in the root directory
|
| 214 |
+
else:
|
| 215 |
+
run_name = dir_name # Use directory name
|
| 216 |
+
|
| 217 |
+
if name:
|
| 218 |
+
run_name = f"{name}_{run_name}"
|
| 219 |
+
|
| 220 |
+
if group_df.empty:
|
| 221 |
+
print(f"* Skipping directory {dir_name}: no scalar data found")
|
| 222 |
+
continue
|
| 223 |
+
|
| 224 |
+
metrics_list = []
|
| 225 |
+
steps = []
|
| 226 |
+
timestamps = []
|
| 227 |
+
|
| 228 |
+
for _, row in group_df.iterrows():
|
| 229 |
+
# Convert row values to appropriate types
|
| 230 |
+
tag = str(row["tag"])
|
| 231 |
+
value = float(row["value"])
|
| 232 |
+
step = int(row["step"])
|
| 233 |
+
|
| 234 |
+
metrics = {tag: value}
|
| 235 |
+
metrics_list.append(metrics)
|
| 236 |
+
steps.append(step)
|
| 237 |
+
|
| 238 |
+
# Use wall_time if present, else fallback
|
| 239 |
+
if "wall_time" in group_df.columns and not bool(
|
| 240 |
+
pd.isna(row["wall_time"])
|
| 241 |
+
):
|
| 242 |
+
timestamps.append(str(row["wall_time"]))
|
| 243 |
+
else:
|
| 244 |
+
timestamps.append("")
|
| 245 |
+
|
| 246 |
+
if metrics_list:
|
| 247 |
+
SQLiteStorage.bulk_log(
|
| 248 |
+
project=project,
|
| 249 |
+
run=str(run_name),
|
| 250 |
+
metrics_list=metrics_list,
|
| 251 |
+
steps=steps,
|
| 252 |
+
timestamps=timestamps,
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
total_imported += len(metrics_list)
|
| 256 |
+
imported_runs.append(run_name)
|
| 257 |
+
|
| 258 |
+
print(
|
| 259 |
+
f"* Imported {len(metrics_list)} scalar events from directory '{dir_name}' as run '{run_name}'"
|
| 260 |
+
)
|
| 261 |
+
print(f"* Metrics in this run: {', '.join(set(group_df['tag']))}")
|
| 262 |
+
|
| 263 |
+
except Exception as e:
|
| 264 |
+
print(f"* Error processing directory {dir_name}: {e}")
|
| 265 |
+
continue
|
| 266 |
+
|
| 267 |
+
if not imported_runs:
|
| 268 |
+
raise ValueError("No valid TensorFlow events data could be imported")
|
| 269 |
+
|
| 270 |
+
print(f"* Total imported events: {total_imported}")
|
| 271 |
+
print(f"* Created runs: {', '.join(imported_runs)}")
|
| 272 |
+
|
| 273 |
+
space_id, dataset_id, _ = utils.preprocess_space_and_dataset_ids(
|
| 274 |
+
space_id, dataset_id
|
| 275 |
+
)
|
| 276 |
+
if dataset_id is not None:
|
| 277 |
+
os.environ["TRACKIO_DATASET_ID"] = dataset_id
|
| 278 |
+
print(f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}")
|
| 279 |
+
|
| 280 |
+
if space_id is None:
|
| 281 |
+
utils.print_dashboard_instructions(project)
|
| 282 |
+
else:
|
| 283 |
+
deploy.create_space_if_not_exists(
|
| 284 |
+
space_id, dataset_id=dataset_id, private=private
|
| 285 |
+
)
|
| 286 |
+
deploy.wait_until_space_exists(space_id)
|
| 287 |
+
deploy.upload_db_to_space(project, space_id, force=force)
|
| 288 |
+
print(
|
| 289 |
+
f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}"
|
| 290 |
+
)
|
trackio/launch.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import secrets
|
| 5 |
+
import socket
|
| 6 |
+
import threading
|
| 7 |
+
import time
|
| 8 |
+
import warnings
|
| 9 |
+
from typing import Any
|
| 10 |
+
|
| 11 |
+
import httpx
|
| 12 |
+
import uvicorn
|
| 13 |
+
from uvicorn.config import Config
|
| 14 |
+
|
| 15 |
+
from trackio.launch_utils import colab_check, is_hosted_notebook
|
| 16 |
+
|
| 17 |
+
INITIAL_PORT_VALUE = int(os.getenv("GRADIO_SERVER_PORT", "7860"))
|
| 18 |
+
TRY_NUM_PORTS = int(os.getenv("GRADIO_NUM_PORTS", "100"))
|
| 19 |
+
LOCALHOST_NAME = os.getenv("GRADIO_SERVER_NAME", "127.0.0.1")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class _UvicornServer(uvicorn.Server):
|
| 23 |
+
def install_signal_handlers(self) -> None:
|
| 24 |
+
pass
|
| 25 |
+
|
| 26 |
+
def run_in_thread(self) -> None:
|
| 27 |
+
self.thread = threading.Thread(target=self.run, daemon=True)
|
| 28 |
+
self.thread.start()
|
| 29 |
+
start = time.time()
|
| 30 |
+
while not self.started:
|
| 31 |
+
time.sleep(1e-3)
|
| 32 |
+
if time.time() - start > 60:
|
| 33 |
+
raise RuntimeError(
|
| 34 |
+
"Server failed to start. Please check that the port is available."
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _bind_host(server_name: str) -> str:
|
| 39 |
+
if server_name.startswith("[") and server_name.endswith("]"):
|
| 40 |
+
return server_name[1:-1]
|
| 41 |
+
return server_name
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def start_server(
|
| 45 |
+
app: Any,
|
| 46 |
+
server_name: str | None = None,
|
| 47 |
+
server_port: int | None = None,
|
| 48 |
+
ssl_keyfile: str | None = None,
|
| 49 |
+
ssl_certfile: str | None = None,
|
| 50 |
+
ssl_keyfile_password: str | None = None,
|
| 51 |
+
) -> tuple[str, int, str, _UvicornServer]:
|
| 52 |
+
server_name = server_name or LOCALHOST_NAME
|
| 53 |
+
url_host_name = "localhost" if server_name == "0.0.0.0" else server_name
|
| 54 |
+
|
| 55 |
+
host = _bind_host(server_name)
|
| 56 |
+
|
| 57 |
+
server_ports = (
|
| 58 |
+
[server_port]
|
| 59 |
+
if server_port is not None
|
| 60 |
+
else range(INITIAL_PORT_VALUE, INITIAL_PORT_VALUE + TRY_NUM_PORTS)
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
port_used = None
|
| 64 |
+
server = None
|
| 65 |
+
for port in server_ports:
|
| 66 |
+
try:
|
| 67 |
+
s = socket.socket()
|
| 68 |
+
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
| 69 |
+
s.bind((LOCALHOST_NAME, port))
|
| 70 |
+
s.close()
|
| 71 |
+
config = Config(
|
| 72 |
+
app=app,
|
| 73 |
+
port=port,
|
| 74 |
+
host=host,
|
| 75 |
+
log_level="warning",
|
| 76 |
+
ssl_keyfile=ssl_keyfile,
|
| 77 |
+
ssl_certfile=ssl_certfile,
|
| 78 |
+
ssl_keyfile_password=ssl_keyfile_password,
|
| 79 |
+
)
|
| 80 |
+
server = _UvicornServer(config=config)
|
| 81 |
+
server.run_in_thread()
|
| 82 |
+
port_used = port
|
| 83 |
+
break
|
| 84 |
+
except (OSError, RuntimeError):
|
| 85 |
+
continue
|
| 86 |
+
else:
|
| 87 |
+
raise OSError(
|
| 88 |
+
f"Cannot find empty port in range: {min(server_ports)}-{max(server_ports)}. "
|
| 89 |
+
"Set GRADIO_SERVER_PORT or pass server_port to trackio.show()."
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
assert port_used is not None and server is not None
|
| 93 |
+
|
| 94 |
+
if ssl_keyfile is not None:
|
| 95 |
+
path_to_local_server = f"https://{url_host_name}:{port_used}/"
|
| 96 |
+
else:
|
| 97 |
+
path_to_local_server = f"http://{url_host_name}:{port_used}/"
|
| 98 |
+
|
| 99 |
+
return server_name, port_used, path_to_local_server, server
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def launch_trackio_dashboard(
|
| 103 |
+
starlette_app: Any,
|
| 104 |
+
*,
|
| 105 |
+
server_name: str | None = None,
|
| 106 |
+
server_port: int | None = None,
|
| 107 |
+
share: bool | None = None,
|
| 108 |
+
share_server_address: str | None = None,
|
| 109 |
+
share_server_protocol: str | None = None,
|
| 110 |
+
share_server_tls_certificate: str | None = None,
|
| 111 |
+
mcp_server: bool = False,
|
| 112 |
+
ssl_verify: bool = True,
|
| 113 |
+
quiet: bool = False,
|
| 114 |
+
) -> tuple[str | None, str | None, str | None, Any]:
|
| 115 |
+
from pathlib import Path
|
| 116 |
+
|
| 117 |
+
from trackio._vendor.networking import normalize_share_url, setup_tunnel
|
| 118 |
+
from trackio._vendor.tunneling import BINARY_PATH
|
| 119 |
+
|
| 120 |
+
is_colab = colab_check()
|
| 121 |
+
is_hosted_nb = is_hosted_notebook()
|
| 122 |
+
space_id = os.getenv("SPACE_ID")
|
| 123 |
+
|
| 124 |
+
if share is None:
|
| 125 |
+
if is_colab or is_hosted_nb:
|
| 126 |
+
if not quiet:
|
| 127 |
+
print(
|
| 128 |
+
"It looks like you are running Trackio on a hosted Jupyter notebook, which requires "
|
| 129 |
+
"`share=True`. Automatically setting `share=True` "
|
| 130 |
+
"(set `share=False` in `show()` to disable).\n"
|
| 131 |
+
)
|
| 132 |
+
share = True
|
| 133 |
+
else:
|
| 134 |
+
share = os.getenv("GRADIO_SHARE", "").lower() == "true"
|
| 135 |
+
|
| 136 |
+
sn = server_name
|
| 137 |
+
if sn is None and os.getenv("SYSTEM") == "spaces":
|
| 138 |
+
sn = "0.0.0.0"
|
| 139 |
+
elif sn is None:
|
| 140 |
+
sn = LOCALHOST_NAME
|
| 141 |
+
|
| 142 |
+
server_name_r, server_port_r, local_url, uv_server = start_server(
|
| 143 |
+
starlette_app,
|
| 144 |
+
server_name=sn,
|
| 145 |
+
server_port=server_port,
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
local_api_url = f"{local_url.rstrip('/')}/gradio_api/"
|
| 149 |
+
try:
|
| 150 |
+
httpx.get(f"{local_api_url}startup-events", verify=ssl_verify, timeout=10)
|
| 151 |
+
except Exception as e:
|
| 152 |
+
raise RuntimeError(
|
| 153 |
+
f"Could not reach startup-events at {local_api_url}startup-events: {e}"
|
| 154 |
+
) from e
|
| 155 |
+
|
| 156 |
+
if share and space_id:
|
| 157 |
+
warnings.warn("Setting share=True is not supported on Hugging Face Spaces")
|
| 158 |
+
share = False
|
| 159 |
+
|
| 160 |
+
share_url: str | None = None
|
| 161 |
+
if share:
|
| 162 |
+
try:
|
| 163 |
+
share_tok = secrets.token_urlsafe(32)
|
| 164 |
+
proto = share_server_protocol or (
|
| 165 |
+
"http" if share_server_address is not None else "https"
|
| 166 |
+
)
|
| 167 |
+
raw = setup_tunnel(
|
| 168 |
+
local_host=server_name_r,
|
| 169 |
+
local_port=server_port_r,
|
| 170 |
+
share_token=share_tok,
|
| 171 |
+
share_server_address=share_server_address,
|
| 172 |
+
share_server_tls_certificate=share_server_tls_certificate,
|
| 173 |
+
)
|
| 174 |
+
share_url = normalize_share_url(raw, proto)
|
| 175 |
+
if not quiet:
|
| 176 |
+
print(f"* Running on public URL: {share_url}")
|
| 177 |
+
print(
|
| 178 |
+
"\nThis share link expires in 1 week. For permanent hosting, deploy to Hugging Face Spaces."
|
| 179 |
+
)
|
| 180 |
+
except Exception as e:
|
| 181 |
+
share_url = None
|
| 182 |
+
if not quiet:
|
| 183 |
+
from trackio._vendor.gradio_exceptions import ChecksumMismatchError
|
| 184 |
+
|
| 185 |
+
if isinstance(e, ChecksumMismatchError):
|
| 186 |
+
print(
|
| 187 |
+
"\nCould not create share link. Checksum mismatch for frpc binary."
|
| 188 |
+
)
|
| 189 |
+
elif Path(BINARY_PATH).exists():
|
| 190 |
+
print(
|
| 191 |
+
"\nCould not create share link. Check your internet connection or https://status.gradio.app."
|
| 192 |
+
)
|
| 193 |
+
else:
|
| 194 |
+
print(
|
| 195 |
+
f"\nCould not create share link. Missing frpc at {BINARY_PATH}. {e}"
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
if not share_url and not quiet:
|
| 199 |
+
print("* To create a public link, set `share=True` in `trackio.show()`.")
|
| 200 |
+
|
| 201 |
+
if mcp_server and not quiet:
|
| 202 |
+
base = share_url or local_url.rstrip("/")
|
| 203 |
+
print(f"\n* MCP streamable HTTP: {base}/gradio_api/mcp/")
|
| 204 |
+
|
| 205 |
+
return local_url, share_url, local_api_url, uv_server
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def url_ok_local(local_url: str) -> bool:
|
| 209 |
+
from trackio._vendor.networking import url_ok
|
| 210 |
+
|
| 211 |
+
return url_ok(local_url)
|
trackio/launch_utils.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def colab_check() -> bool:
|
| 5 |
+
is_colab = False
|
| 6 |
+
try:
|
| 7 |
+
from IPython.core.getipython import get_ipython
|
| 8 |
+
|
| 9 |
+
from_ipynb = get_ipython()
|
| 10 |
+
if "google.colab" in str(from_ipynb):
|
| 11 |
+
is_colab = True
|
| 12 |
+
except (ImportError, NameError):
|
| 13 |
+
pass
|
| 14 |
+
return is_colab
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def is_hosted_notebook() -> bool:
|
| 18 |
+
return bool(
|
| 19 |
+
os.environ.get("KAGGLE_KERNEL_RUN_TYPE")
|
| 20 |
+
or os.path.exists("/home/ec2-user/SageMaker")
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def ipython_check() -> bool:
|
| 25 |
+
is_ipython = False
|
| 26 |
+
try:
|
| 27 |
+
from IPython.core.getipython import get_ipython
|
| 28 |
+
|
| 29 |
+
if get_ipython() is not None:
|
| 30 |
+
is_ipython = True
|
| 31 |
+
except (ImportError, NameError):
|
| 32 |
+
pass
|
| 33 |
+
return is_ipython
|
trackio/markdown.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
class Markdown:
|
| 2 |
+
"""
|
| 3 |
+
Markdown report data type for Trackio.
|
| 4 |
+
|
| 5 |
+
Args:
|
| 6 |
+
text (`str`):
|
| 7 |
+
Markdown content to log.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
TYPE = "trackio.markdown"
|
| 11 |
+
|
| 12 |
+
def __init__(self, text: str = ""):
|
| 13 |
+
if not isinstance(text, str):
|
| 14 |
+
raise ValueError("Markdown text must be a string")
|
| 15 |
+
self.text = text
|
| 16 |
+
|
| 17 |
+
def _to_dict(self) -> dict:
|
| 18 |
+
return {
|
| 19 |
+
"_type": self.TYPE,
|
| 20 |
+
"_value": self.text,
|
| 21 |
+
}
|
trackio/mcp_setup.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from contextlib import asynccontextmanager
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
@asynccontextmanager
|
| 5 |
+
async def mcp_lifespan_context(app):
|
| 6 |
+
yield
|
trackio/media/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Media module for Trackio.
|
| 3 |
+
|
| 4 |
+
This module contains all media-related functionality including:
|
| 5 |
+
- TrackioImage, TrackioVideo, TrackioAudio classes
|
| 6 |
+
- Video writing utilities
|
| 7 |
+
- Audio conversion utilities
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from trackio.media.audio import TrackioAudio
|
| 11 |
+
from trackio.media.image import TrackioImage
|
| 12 |
+
from trackio.media.media import TrackioMedia
|
| 13 |
+
from trackio.media.utils import get_project_media_path
|
| 14 |
+
from trackio.media.video import TrackioVideo
|
| 15 |
+
|
| 16 |
+
write_audio = TrackioAudio.write_audio
|
| 17 |
+
write_video = TrackioVideo.write_video
|
| 18 |
+
|
| 19 |
+
__all__ = [
|
| 20 |
+
"TrackioMedia",
|
| 21 |
+
"TrackioImage",
|
| 22 |
+
"TrackioVideo",
|
| 23 |
+
"TrackioAudio",
|
| 24 |
+
"get_project_media_path",
|
| 25 |
+
"write_video",
|
| 26 |
+
"write_audio",
|
| 27 |
+
]
|
trackio/media/audio.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
import warnings
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Literal
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
from pydub import AudioSegment
|
| 9 |
+
|
| 10 |
+
from trackio.media.media import TrackioMedia
|
| 11 |
+
from trackio.media.utils import check_ffmpeg_installed, check_path
|
| 12 |
+
|
| 13 |
+
SUPPORTED_FORMATS = ["wav", "mp3"]
|
| 14 |
+
AudioFormatType = Literal["wav", "mp3"]
|
| 15 |
+
TrackioAudioSourceType = str | Path | np.ndarray
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class TrackioAudio(TrackioMedia):
|
| 19 |
+
"""
|
| 20 |
+
Initializes an Audio object.
|
| 21 |
+
|
| 22 |
+
Example:
|
| 23 |
+
```python
|
| 24 |
+
import trackio
|
| 25 |
+
import numpy as np
|
| 26 |
+
|
| 27 |
+
# Generate a 1-second 440 Hz sine wave (mono)
|
| 28 |
+
sr = 16000
|
| 29 |
+
t = np.linspace(0, 1, sr, endpoint=False)
|
| 30 |
+
wave = 0.2 * np.sin(2 * np.pi * 440 * t)
|
| 31 |
+
audio = trackio.Audio(wave, caption="A4 sine", sample_rate=sr, format="wav")
|
| 32 |
+
trackio.log({"tone": audio})
|
| 33 |
+
|
| 34 |
+
# Stereo from numpy array (shape: samples, 2)
|
| 35 |
+
stereo = np.stack([wave, wave], axis=1)
|
| 36 |
+
audio = trackio.Audio(stereo, caption="Stereo", sample_rate=sr, format="mp3")
|
| 37 |
+
trackio.log({"stereo": audio})
|
| 38 |
+
|
| 39 |
+
# From an existing file
|
| 40 |
+
audio = trackio.Audio("path/to/audio.wav", caption="From file")
|
| 41 |
+
trackio.log({"file_audio": audio})
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
Args:
|
| 45 |
+
value (`str`, `Path`, or `numpy.ndarray`, *optional*):
|
| 46 |
+
A path to an audio file, or a numpy array.
|
| 47 |
+
The array should be shaped `(samples,)` for mono or `(samples, 2)` for stereo.
|
| 48 |
+
Float arrays will be peak-normalized and converted to 16-bit PCM; integer arrays will be converted to 16-bit PCM as needed.
|
| 49 |
+
caption (`str`, *optional*):
|
| 50 |
+
A string caption for the audio.
|
| 51 |
+
sample_rate (`int`, *optional*):
|
| 52 |
+
Sample rate in Hz. Required when `value` is a numpy array.
|
| 53 |
+
format (`Literal["wav", "mp3"]`, *optional*):
|
| 54 |
+
Audio format used when `value` is a numpy array. Default is "wav".
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
TYPE = "trackio.audio"
|
| 58 |
+
|
| 59 |
+
def __init__(
|
| 60 |
+
self,
|
| 61 |
+
value: TrackioAudioSourceType,
|
| 62 |
+
caption: str | None = None,
|
| 63 |
+
sample_rate: int | None = None,
|
| 64 |
+
format: AudioFormatType | None = None,
|
| 65 |
+
):
|
| 66 |
+
super().__init__(value, caption)
|
| 67 |
+
if isinstance(value, np.ndarray):
|
| 68 |
+
if sample_rate is None:
|
| 69 |
+
raise ValueError("Sample rate is required when value is an ndarray")
|
| 70 |
+
if format is None:
|
| 71 |
+
format = "wav"
|
| 72 |
+
self._format = format
|
| 73 |
+
self._sample_rate = sample_rate
|
| 74 |
+
|
| 75 |
+
def _save_media(self, file_path: Path):
|
| 76 |
+
if isinstance(self._value, np.ndarray):
|
| 77 |
+
TrackioAudio.write_audio(
|
| 78 |
+
data=self._value,
|
| 79 |
+
sample_rate=self._sample_rate,
|
| 80 |
+
filename=file_path,
|
| 81 |
+
format=self._format,
|
| 82 |
+
)
|
| 83 |
+
elif isinstance(self._value, str | Path):
|
| 84 |
+
if os.path.isfile(self._value):
|
| 85 |
+
shutil.copy(self._value, file_path)
|
| 86 |
+
else:
|
| 87 |
+
raise ValueError(f"File not found: {self._value}")
|
| 88 |
+
|
| 89 |
+
@staticmethod
|
| 90 |
+
def ensure_int16_pcm(data: np.ndarray) -> np.ndarray:
|
| 91 |
+
"""
|
| 92 |
+
Convert input audio array to contiguous int16 PCM.
|
| 93 |
+
Peak normalization is applied to floating inputs.
|
| 94 |
+
"""
|
| 95 |
+
arr = np.asarray(data)
|
| 96 |
+
if arr.ndim not in (1, 2):
|
| 97 |
+
raise ValueError("Audio data must be 1D (mono) or 2D ([samples, channels])")
|
| 98 |
+
|
| 99 |
+
if arr.dtype != np.int16:
|
| 100 |
+
warnings.warn(
|
| 101 |
+
f"Converting {arr.dtype} audio to int16 PCM; pass int16 to avoid conversion.",
|
| 102 |
+
stacklevel=2,
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
arr = np.nan_to_num(arr, copy=False)
|
| 106 |
+
|
| 107 |
+
# Floating types: normalize to peak 1.0, then scale to int16
|
| 108 |
+
if np.issubdtype(arr.dtype, np.floating):
|
| 109 |
+
max_abs = float(np.max(np.abs(arr))) if arr.size else 0.0
|
| 110 |
+
if max_abs > 0.0:
|
| 111 |
+
arr = arr / max_abs
|
| 112 |
+
out = (arr * 32767.0).clip(-32768, 32767).astype(np.int16, copy=False)
|
| 113 |
+
return np.ascontiguousarray(out)
|
| 114 |
+
|
| 115 |
+
converters: dict[np.dtype, callable] = {
|
| 116 |
+
np.dtype(np.int16): lambda a: a,
|
| 117 |
+
np.dtype(np.int32): lambda a: (a.astype(np.int32) // 65536).astype(
|
| 118 |
+
np.int16, copy=False
|
| 119 |
+
),
|
| 120 |
+
np.dtype(np.uint16): lambda a: (a.astype(np.int32) - 32768).astype(
|
| 121 |
+
np.int16, copy=False
|
| 122 |
+
),
|
| 123 |
+
np.dtype(np.uint8): lambda a: (a.astype(np.int32) * 257 - 32768).astype(
|
| 124 |
+
np.int16, copy=False
|
| 125 |
+
),
|
| 126 |
+
np.dtype(np.int8): lambda a: (a.astype(np.int32) * 256).astype(
|
| 127 |
+
np.int16, copy=False
|
| 128 |
+
),
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
conv = converters.get(arr.dtype)
|
| 132 |
+
if conv is not None:
|
| 133 |
+
out = conv(arr)
|
| 134 |
+
return np.ascontiguousarray(out)
|
| 135 |
+
raise TypeError(f"Unsupported audio dtype: {arr.dtype}")
|
| 136 |
+
|
| 137 |
+
@staticmethod
|
| 138 |
+
def write_audio(
|
| 139 |
+
data: np.ndarray,
|
| 140 |
+
sample_rate: int,
|
| 141 |
+
filename: str | Path,
|
| 142 |
+
format: AudioFormatType = "wav",
|
| 143 |
+
) -> None:
|
| 144 |
+
if not isinstance(sample_rate, int) or sample_rate <= 0:
|
| 145 |
+
raise ValueError(f"Invalid sample_rate: {sample_rate}")
|
| 146 |
+
if format not in SUPPORTED_FORMATS:
|
| 147 |
+
raise ValueError(
|
| 148 |
+
f"Unsupported format: {format}. Supported: {SUPPORTED_FORMATS}"
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
check_path(filename)
|
| 152 |
+
|
| 153 |
+
pcm = TrackioAudio.ensure_int16_pcm(data)
|
| 154 |
+
|
| 155 |
+
if format != "wav":
|
| 156 |
+
check_ffmpeg_installed()
|
| 157 |
+
|
| 158 |
+
channels = 1 if pcm.ndim == 1 else pcm.shape[1]
|
| 159 |
+
audio = AudioSegment(
|
| 160 |
+
pcm.tobytes(),
|
| 161 |
+
frame_rate=sample_rate,
|
| 162 |
+
sample_width=2, # int16
|
| 163 |
+
channels=channels,
|
| 164 |
+
)
|
| 165 |
+
|
| 166 |
+
file = audio.export(str(filename), format=format)
|
| 167 |
+
file.close()
|
trackio/media/image.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
from PIL import Image as PILImage
|
| 7 |
+
|
| 8 |
+
from trackio.media.media import TrackioMedia
|
| 9 |
+
|
| 10 |
+
TrackioImageSourceType = str | Path | np.ndarray | PILImage.Image
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class TrackioImage(TrackioMedia):
|
| 14 |
+
"""
|
| 15 |
+
Initializes an Image object.
|
| 16 |
+
|
| 17 |
+
Example:
|
| 18 |
+
```python
|
| 19 |
+
import trackio
|
| 20 |
+
import numpy as np
|
| 21 |
+
from PIL import Image
|
| 22 |
+
|
| 23 |
+
# Create an image from numpy array
|
| 24 |
+
image_data = np.random.randint(0, 255, (64, 64, 3), dtype=np.uint8)
|
| 25 |
+
image = trackio.Image(image_data, caption="Random image")
|
| 26 |
+
trackio.log({"my_image": image})
|
| 27 |
+
|
| 28 |
+
# Create an image from PIL Image
|
| 29 |
+
pil_image = Image.new('RGB', (100, 100), color='red')
|
| 30 |
+
image = trackio.Image(pil_image, caption="Red square")
|
| 31 |
+
trackio.log({"red_image": image})
|
| 32 |
+
|
| 33 |
+
# Create an image from file path
|
| 34 |
+
image = trackio.Image("path/to/image.jpg", caption="Photo from file")
|
| 35 |
+
trackio.log({"file_image": image})
|
| 36 |
+
```
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
value (`str`, `Path`, `numpy.ndarray`, or `PIL.Image`, *optional*):
|
| 40 |
+
A path to an image, a PIL Image, or a numpy array of shape (height, width, channels).
|
| 41 |
+
If numpy array, should be of type `np.uint8` with RGB values in the range `[0, 255]`.
|
| 42 |
+
caption (`str`, *optional*):
|
| 43 |
+
A string caption for the image.
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
TYPE = "trackio.image"
|
| 47 |
+
|
| 48 |
+
def __init__(self, value: TrackioImageSourceType, caption: str | None = None):
|
| 49 |
+
super().__init__(value, caption)
|
| 50 |
+
self._format: str | None = None
|
| 51 |
+
|
| 52 |
+
if not isinstance(self._value, TrackioImageSourceType):
|
| 53 |
+
raise ValueError(
|
| 54 |
+
f"Invalid value type, expected {TrackioImageSourceType}, got {type(self._value)}"
|
| 55 |
+
)
|
| 56 |
+
if isinstance(self._value, np.ndarray) and self._value.dtype != np.uint8:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
f"Invalid value dtype, expected np.uint8, got {self._value.dtype}"
|
| 59 |
+
)
|
| 60 |
+
if (
|
| 61 |
+
isinstance(self._value, np.ndarray | PILImage.Image)
|
| 62 |
+
and self._format is None
|
| 63 |
+
):
|
| 64 |
+
self._format = "png"
|
| 65 |
+
|
| 66 |
+
def _as_pil(self) -> PILImage.Image | None:
|
| 67 |
+
try:
|
| 68 |
+
if isinstance(self._value, np.ndarray):
|
| 69 |
+
arr = np.asarray(self._value).astype("uint8")
|
| 70 |
+
return PILImage.fromarray(arr).convert("RGBA")
|
| 71 |
+
if isinstance(self._value, PILImage.Image):
|
| 72 |
+
return self._value.convert("RGBA")
|
| 73 |
+
except Exception as e:
|
| 74 |
+
raise ValueError(f"Failed to process image data: {self._value}") from e
|
| 75 |
+
return None
|
| 76 |
+
|
| 77 |
+
def _save_media(self, file_path: Path):
|
| 78 |
+
if pil := self._as_pil():
|
| 79 |
+
pil.save(file_path, format=self._format)
|
| 80 |
+
elif isinstance(self._value, str | Path):
|
| 81 |
+
if os.path.isfile(self._value):
|
| 82 |
+
shutil.copy(self._value, file_path)
|
| 83 |
+
else:
|
| 84 |
+
raise ValueError(f"File not found: {self._value}")
|
trackio/media/media.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import uuid
|
| 3 |
+
from abc import ABC, abstractmethod
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
from trackio.media.utils import get_project_media_path
|
| 7 |
+
from trackio.utils import MEDIA_DIR
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class TrackioMedia(ABC):
|
| 11 |
+
"""
|
| 12 |
+
Abstract base class for Trackio media objects
|
| 13 |
+
Provides shared functionality for file handling and serialization.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
TYPE: str
|
| 17 |
+
|
| 18 |
+
def __init_subclass__(cls, **kwargs):
|
| 19 |
+
"""Ensure subclasses define the TYPE attribute."""
|
| 20 |
+
super().__init_subclass__(**kwargs)
|
| 21 |
+
if not hasattr(cls, "TYPE") or cls.TYPE is None:
|
| 22 |
+
raise TypeError(f"Class {cls.__name__} must define TYPE attribute")
|
| 23 |
+
|
| 24 |
+
def __init__(self, value, caption: str | None = None):
|
| 25 |
+
"""
|
| 26 |
+
Saves the value and caption, and if the value is a file path, checks if the file exists.
|
| 27 |
+
"""
|
| 28 |
+
self.caption = caption
|
| 29 |
+
self._value = value
|
| 30 |
+
self._file_path: Path | None = None
|
| 31 |
+
|
| 32 |
+
if isinstance(self._value, str | Path):
|
| 33 |
+
if not os.path.isfile(self._value):
|
| 34 |
+
raise ValueError(f"File not found: {self._value}")
|
| 35 |
+
|
| 36 |
+
def _file_extension(self) -> str:
|
| 37 |
+
if self._file_path:
|
| 38 |
+
return self._file_path.suffix[1:].lower()
|
| 39 |
+
if isinstance(self._value, str | Path):
|
| 40 |
+
path = Path(self._value)
|
| 41 |
+
return path.suffix[1:].lower()
|
| 42 |
+
if hasattr(self, "_format") and self._format:
|
| 43 |
+
return self._format
|
| 44 |
+
return "unknown"
|
| 45 |
+
|
| 46 |
+
def _get_relative_file_path(self) -> Path | None:
|
| 47 |
+
return self._file_path
|
| 48 |
+
|
| 49 |
+
def _get_absolute_file_path(self) -> Path | None:
|
| 50 |
+
if self._file_path:
|
| 51 |
+
return MEDIA_DIR / self._file_path
|
| 52 |
+
return None
|
| 53 |
+
|
| 54 |
+
def _save(self, project: str, run: str, step: int = 0):
|
| 55 |
+
if self._file_path:
|
| 56 |
+
return
|
| 57 |
+
|
| 58 |
+
media_dir = get_project_media_path(project=project, run=run, step=step)
|
| 59 |
+
filename = f"{uuid.uuid4()}.{self._file_extension()}"
|
| 60 |
+
file_path = media_dir / filename
|
| 61 |
+
|
| 62 |
+
self._save_media(file_path)
|
| 63 |
+
self._file_path = file_path.relative_to(MEDIA_DIR)
|
| 64 |
+
|
| 65 |
+
@abstractmethod
|
| 66 |
+
def _save_media(self, file_path: Path):
|
| 67 |
+
"""
|
| 68 |
+
Performs the actual media saving logic.
|
| 69 |
+
"""
|
| 70 |
+
pass
|
| 71 |
+
|
| 72 |
+
def _to_dict(self) -> dict:
|
| 73 |
+
if not self._file_path:
|
| 74 |
+
raise ValueError("Media must be saved to file before serialization")
|
| 75 |
+
return {
|
| 76 |
+
"_type": self.TYPE,
|
| 77 |
+
"file_path": str(self._get_relative_file_path()),
|
| 78 |
+
"caption": self.caption,
|
| 79 |
+
}
|
trackio/media/utils.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import shutil
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
from trackio.utils import MEDIA_DIR
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def check_path(file_path: str | Path) -> None:
|
| 8 |
+
"""Raise an error if the parent directory does not exist."""
|
| 9 |
+
file_path = Path(file_path)
|
| 10 |
+
if not file_path.parent.exists():
|
| 11 |
+
try:
|
| 12 |
+
file_path.parent.mkdir(parents=True, exist_ok=True)
|
| 13 |
+
except OSError as e:
|
| 14 |
+
raise ValueError(
|
| 15 |
+
f"Failed to create parent directory {file_path.parent}: {e}"
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def check_ffmpeg_installed() -> None:
|
| 20 |
+
"""Raise an error if ffmpeg is not available on the system PATH."""
|
| 21 |
+
if shutil.which("ffmpeg") is None:
|
| 22 |
+
raise RuntimeError(
|
| 23 |
+
"ffmpeg is required to write video but was not found on your system. "
|
| 24 |
+
"Please install ffmpeg and ensure it is available on your PATH."
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def get_project_media_path(
|
| 29 |
+
project: str,
|
| 30 |
+
run: str | None = None,
|
| 31 |
+
step: int | None = None,
|
| 32 |
+
relative_path: str | Path | None = None,
|
| 33 |
+
) -> Path:
|
| 34 |
+
"""
|
| 35 |
+
Get the full path where uploaded files are stored for a Trackio project (and create the directory if it doesn't exist).
|
| 36 |
+
If a run is not provided, the files are stored in a project-level directory with the given relative path.
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
project: The project name
|
| 40 |
+
run: The run name
|
| 41 |
+
step: The step number
|
| 42 |
+
relative_path: The relative path within the directory (only used if run is not provided)
|
| 43 |
+
|
| 44 |
+
Returns:
|
| 45 |
+
The full path to the media file
|
| 46 |
+
"""
|
| 47 |
+
if step is not None and run is None:
|
| 48 |
+
raise ValueError("Uploading files at a specific step requires a run")
|
| 49 |
+
|
| 50 |
+
path = MEDIA_DIR / project
|
| 51 |
+
if run:
|
| 52 |
+
path /= run
|
| 53 |
+
if step is not None:
|
| 54 |
+
path /= str(step)
|
| 55 |
+
else:
|
| 56 |
+
path /= "files"
|
| 57 |
+
if relative_path:
|
| 58 |
+
path /= relative_path
|
| 59 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 60 |
+
return path
|
trackio/media/video.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
import subprocess
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Literal
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
|
| 9 |
+
from trackio.media.media import TrackioMedia
|
| 10 |
+
from trackio.media.utils import check_ffmpeg_installed, check_path
|
| 11 |
+
|
| 12 |
+
TrackioVideoSourceType = str | Path | np.ndarray
|
| 13 |
+
TrackioVideoFormatType = Literal["gif", "mp4", "webm"]
|
| 14 |
+
VideoCodec = Literal["h264", "vp9", "gif"]
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class TrackioVideo(TrackioMedia):
|
| 18 |
+
"""
|
| 19 |
+
Initializes a Video object.
|
| 20 |
+
|
| 21 |
+
Example:
|
| 22 |
+
```python
|
| 23 |
+
import trackio
|
| 24 |
+
import numpy as np
|
| 25 |
+
|
| 26 |
+
# Create a simple video from numpy array
|
| 27 |
+
frames = np.random.randint(0, 255, (10, 3, 64, 64), dtype=np.uint8)
|
| 28 |
+
video = trackio.Video(frames, caption="Random video", fps=30)
|
| 29 |
+
|
| 30 |
+
# Create a batch of videos
|
| 31 |
+
batch_frames = np.random.randint(0, 255, (3, 10, 3, 64, 64), dtype=np.uint8)
|
| 32 |
+
batch_video = trackio.Video(batch_frames, caption="Batch of videos", fps=15)
|
| 33 |
+
|
| 34 |
+
# Create video from file path
|
| 35 |
+
video = trackio.Video("path/to/video.mp4", caption="Video from file")
|
| 36 |
+
```
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
value (`str`, `Path`, or `numpy.ndarray`, *optional*):
|
| 40 |
+
A path to a video file, or a numpy array.
|
| 41 |
+
If numpy array, should be of type `np.uint8` with RGB values in the range `[0, 255]`.
|
| 42 |
+
It is expected to have shape of either (frames, channels, height, width) or (batch, frames, channels, height, width).
|
| 43 |
+
For the latter, the videos will be tiled into a grid.
|
| 44 |
+
caption (`str`, *optional*):
|
| 45 |
+
A string caption for the video.
|
| 46 |
+
fps (`int`, *optional*):
|
| 47 |
+
Frames per second for the video. Only used when value is an ndarray. Default is `24`.
|
| 48 |
+
format (`Literal["gif", "mp4", "webm"]`, *optional*):
|
| 49 |
+
Video format ("gif", "mp4", or "webm"). Only used when value is an ndarray. Default is "gif".
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
TYPE = "trackio.video"
|
| 53 |
+
|
| 54 |
+
def __init__(
|
| 55 |
+
self,
|
| 56 |
+
value: TrackioVideoSourceType,
|
| 57 |
+
caption: str | None = None,
|
| 58 |
+
fps: int | None = None,
|
| 59 |
+
format: TrackioVideoFormatType | None = None,
|
| 60 |
+
):
|
| 61 |
+
super().__init__(value, caption)
|
| 62 |
+
|
| 63 |
+
if not isinstance(self._value, TrackioVideoSourceType):
|
| 64 |
+
raise ValueError(
|
| 65 |
+
f"Invalid value type, expected {TrackioVideoSourceType}, got {type(self._value)}"
|
| 66 |
+
)
|
| 67 |
+
if isinstance(self._value, np.ndarray):
|
| 68 |
+
if self._value.dtype != np.uint8:
|
| 69 |
+
raise ValueError(
|
| 70 |
+
f"Invalid value dtype, expected np.uint8, got {self._value.dtype}"
|
| 71 |
+
)
|
| 72 |
+
if format is None:
|
| 73 |
+
format = "gif"
|
| 74 |
+
if fps is None:
|
| 75 |
+
fps = 24
|
| 76 |
+
self._fps = fps
|
| 77 |
+
self._format = format
|
| 78 |
+
|
| 79 |
+
@staticmethod
|
| 80 |
+
def _check_array_format(video: np.ndarray) -> None:
|
| 81 |
+
"""Raise an error if the array is not in the expected format."""
|
| 82 |
+
if not (video.ndim == 4 and video.shape[-1] == 3):
|
| 83 |
+
raise ValueError(
|
| 84 |
+
f"Expected RGB input shaped (F, H, W, 3), got {video.shape}. "
|
| 85 |
+
f"Input has {video.ndim} dimensions, expected 4."
|
| 86 |
+
)
|
| 87 |
+
if video.dtype != np.uint8:
|
| 88 |
+
raise TypeError(
|
| 89 |
+
f"Expected dtype=uint8, got {video.dtype}. "
|
| 90 |
+
"Please convert your video data to uint8 format."
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
@staticmethod
|
| 94 |
+
def write_video(
|
| 95 |
+
file_path: str | Path, video: np.ndarray, fps: float, codec: VideoCodec
|
| 96 |
+
) -> None:
|
| 97 |
+
"""RGB uint8 only, shape (F, H, W, 3)."""
|
| 98 |
+
check_ffmpeg_installed()
|
| 99 |
+
check_path(file_path)
|
| 100 |
+
|
| 101 |
+
if codec not in {"h264", "vp9", "gif"}:
|
| 102 |
+
raise ValueError("Unsupported codec. Use h264, vp9, or gif.")
|
| 103 |
+
|
| 104 |
+
arr = np.asarray(video)
|
| 105 |
+
TrackioVideo._check_array_format(arr)
|
| 106 |
+
|
| 107 |
+
frames = np.ascontiguousarray(arr)
|
| 108 |
+
_, height, width, _ = frames.shape
|
| 109 |
+
out_path = str(file_path)
|
| 110 |
+
|
| 111 |
+
cmd = [
|
| 112 |
+
"ffmpeg",
|
| 113 |
+
"-y",
|
| 114 |
+
"-f",
|
| 115 |
+
"rawvideo",
|
| 116 |
+
"-s",
|
| 117 |
+
f"{width}x{height}",
|
| 118 |
+
"-pix_fmt",
|
| 119 |
+
"rgb24",
|
| 120 |
+
"-r",
|
| 121 |
+
str(fps),
|
| 122 |
+
"-i",
|
| 123 |
+
"-",
|
| 124 |
+
"-an",
|
| 125 |
+
]
|
| 126 |
+
|
| 127 |
+
if codec == "gif":
|
| 128 |
+
video_filter = "split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse"
|
| 129 |
+
cmd += [
|
| 130 |
+
"-vf",
|
| 131 |
+
video_filter,
|
| 132 |
+
"-loop",
|
| 133 |
+
"0",
|
| 134 |
+
]
|
| 135 |
+
elif codec == "h264":
|
| 136 |
+
cmd += [
|
| 137 |
+
"-vcodec",
|
| 138 |
+
"libx264",
|
| 139 |
+
"-pix_fmt",
|
| 140 |
+
"yuv420p",
|
| 141 |
+
"-movflags",
|
| 142 |
+
"+faststart",
|
| 143 |
+
]
|
| 144 |
+
elif codec == "vp9":
|
| 145 |
+
bpp = 0.08
|
| 146 |
+
bps = int(width * height * fps * bpp)
|
| 147 |
+
if bps >= 1_000_000:
|
| 148 |
+
bitrate = f"{round(bps / 1_000_000)}M"
|
| 149 |
+
elif bps >= 1_000:
|
| 150 |
+
bitrate = f"{round(bps / 1_000)}k"
|
| 151 |
+
else:
|
| 152 |
+
bitrate = str(max(bps, 1))
|
| 153 |
+
cmd += [
|
| 154 |
+
"-vcodec",
|
| 155 |
+
"libvpx-vp9",
|
| 156 |
+
"-b:v",
|
| 157 |
+
bitrate,
|
| 158 |
+
"-pix_fmt",
|
| 159 |
+
"yuv420p",
|
| 160 |
+
]
|
| 161 |
+
cmd += [out_path]
|
| 162 |
+
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
| 163 |
+
try:
|
| 164 |
+
for frame in frames:
|
| 165 |
+
proc.stdin.write(frame.tobytes())
|
| 166 |
+
finally:
|
| 167 |
+
if proc.stdin:
|
| 168 |
+
proc.stdin.close()
|
| 169 |
+
stderr = (
|
| 170 |
+
proc.stderr.read().decode("utf-8", errors="ignore")
|
| 171 |
+
if proc.stderr
|
| 172 |
+
else ""
|
| 173 |
+
)
|
| 174 |
+
ret = proc.wait()
|
| 175 |
+
if ret != 0:
|
| 176 |
+
raise RuntimeError(f"ffmpeg failed with code {ret}\n{stderr}")
|
| 177 |
+
|
| 178 |
+
@property
|
| 179 |
+
def _codec(self) -> str:
|
| 180 |
+
match self._format:
|
| 181 |
+
case "gif":
|
| 182 |
+
return "gif"
|
| 183 |
+
case "mp4":
|
| 184 |
+
return "h264"
|
| 185 |
+
case "webm":
|
| 186 |
+
return "vp9"
|
| 187 |
+
case _:
|
| 188 |
+
raise ValueError(f"Unsupported format: {self._format}")
|
| 189 |
+
|
| 190 |
+
def _save_media(self, file_path: Path):
|
| 191 |
+
if isinstance(self._value, np.ndarray):
|
| 192 |
+
video = TrackioVideo._process_ndarray(self._value)
|
| 193 |
+
TrackioVideo.write_video(file_path, video, fps=self._fps, codec=self._codec)
|
| 194 |
+
elif isinstance(self._value, str | Path):
|
| 195 |
+
if os.path.isfile(self._value):
|
| 196 |
+
shutil.copy(self._value, file_path)
|
| 197 |
+
else:
|
| 198 |
+
raise ValueError(f"File not found: {self._value}")
|
| 199 |
+
|
| 200 |
+
@staticmethod
|
| 201 |
+
def _process_ndarray(value: np.ndarray) -> np.ndarray:
|
| 202 |
+
# Verify value is either 4D (single video) or 5D array (batched videos).
|
| 203 |
+
# Expected format: (frames, channels, height, width) or (batch, frames, channels, height, width)
|
| 204 |
+
if value.ndim < 4:
|
| 205 |
+
raise ValueError(
|
| 206 |
+
"Video requires at least 4 dimensions (frames, channels, height, width)"
|
| 207 |
+
)
|
| 208 |
+
if value.ndim > 5:
|
| 209 |
+
raise ValueError(
|
| 210 |
+
"Videos can have at most 5 dimensions (batch, frames, channels, height, width)"
|
| 211 |
+
)
|
| 212 |
+
if value.ndim == 4:
|
| 213 |
+
# Reshape to 5D with single batch: (1, frames, channels, height, width)
|
| 214 |
+
value = value[np.newaxis, ...]
|
| 215 |
+
|
| 216 |
+
value = TrackioVideo._tile_batched_videos(value)
|
| 217 |
+
return value
|
| 218 |
+
|
| 219 |
+
@staticmethod
|
| 220 |
+
def _tile_batched_videos(video: np.ndarray) -> np.ndarray:
|
| 221 |
+
"""
|
| 222 |
+
Tiles a batch of videos into a grid of videos.
|
| 223 |
+
|
| 224 |
+
Input format: (batch, frames, channels, height, width) - original FCHW format
|
| 225 |
+
Output format: (frames, total_height, total_width, channels)
|
| 226 |
+
"""
|
| 227 |
+
batch_size, frames, channels, height, width = video.shape
|
| 228 |
+
|
| 229 |
+
next_pow2 = 1 << (batch_size - 1).bit_length()
|
| 230 |
+
if batch_size != next_pow2:
|
| 231 |
+
pad_len = next_pow2 - batch_size
|
| 232 |
+
pad_shape = (pad_len, frames, channels, height, width)
|
| 233 |
+
padding = np.zeros(pad_shape, dtype=video.dtype)
|
| 234 |
+
video = np.concatenate((video, padding), axis=0)
|
| 235 |
+
batch_size = next_pow2
|
| 236 |
+
|
| 237 |
+
n_rows = 1 << ((batch_size.bit_length() - 1) // 2)
|
| 238 |
+
n_cols = batch_size // n_rows
|
| 239 |
+
|
| 240 |
+
# Reshape to grid layout: (n_rows, n_cols, frames, channels, height, width)
|
| 241 |
+
video = video.reshape(n_rows, n_cols, frames, channels, height, width)
|
| 242 |
+
|
| 243 |
+
# Rearrange dimensions to (frames, total_height, total_width, channels)
|
| 244 |
+
video = video.transpose(2, 0, 4, 1, 5, 3)
|
| 245 |
+
video = video.reshape(frames, n_rows * height, n_cols * width, channels)
|
| 246 |
+
return video
|
trackio/package.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "trackio",
|
| 3 |
+
"version": "0.22.0",
|
| 4 |
+
"description": "",
|
| 5 |
+
"python": "true"
|
| 6 |
+
}
|
trackio/py.typed
ADDED
|
File without changes
|
trackio/remote_client.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from gradio_client import Client
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class RemoteClient:
|
| 7 |
+
def __init__(self, space: str, hf_token: str | None = None):
|
| 8 |
+
self._space = space
|
| 9 |
+
kwargs: dict = {"verbose": False}
|
| 10 |
+
if hf_token:
|
| 11 |
+
kwargs["hf_token"] = hf_token
|
| 12 |
+
try:
|
| 13 |
+
self._client = Client(space, **kwargs)
|
| 14 |
+
except Exception as e:
|
| 15 |
+
raise ConnectionError(
|
| 16 |
+
f"Could not connect to Space '{space}'. Is it running?\n{e}"
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
def predict(self, *args, api_name: str):
|
| 20 |
+
try:
|
| 21 |
+
return self._client.predict(*args, api_name=api_name)
|
| 22 |
+
except Exception as e:
|
| 23 |
+
if "API Not Found" in str(e) or "api_name" in str(e):
|
| 24 |
+
raise RuntimeError(
|
| 25 |
+
f"Space '{self._space}' does not support '{api_name}'. "
|
| 26 |
+
"Redeploy with `trackio sync`."
|
| 27 |
+
)
|
| 28 |
+
raise
|