Publish tiny_computer_agent_app.py from marimo
Browse files- tiny_computer_agent_app.py +22 -2
tiny_computer_agent_app.py
CHANGED
|
@@ -22,6 +22,7 @@ with app.setup:
|
|
| 22 |
@app.cell(hide_code=True)
|
| 23 |
def readme():
|
| 24 |
readme_md = '''
|
|
|
|
| 25 |
---
|
| 26 |
title: Tiny Computer Agent
|
| 27 |
emoji: 🖱️
|
|
@@ -174,6 +175,21 @@ def readme():
|
|
| 174 |
was actually drawn at, clicks it for real, and checks the app reacted. It is opt-in
|
| 175 |
because it moves your mouse — run it on a throwaway desktop.
|
| 176 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
## How it works
|
| 178 |
|
| 179 |
| Module | Job |
|
|
@@ -234,6 +250,7 @@ def readme():
|
|
| 234 |
- [x] A real click on a real window, with a verified outcome
|
| 235 |
- [x] Blind-grid-click guard, echo detection, no-progress guard
|
| 236 |
- [ ] Make the VLM reliable on a live screen (it echoes the prompt instead of choosing)
|
|
|
|
| 237 |
- [ ] Accessibility-tree detector (AT-SPI on Linux, UI Automation on Windows)
|
| 238 |
- [ ] Score episodes: replay a trace and count how often the choice was right
|
| 239 |
- [ ] marimo console: live screen preview, action log, pause / resume / step
|
|
@@ -380,6 +397,9 @@ def publish_status(
|
|
| 380 |
".gitignore": gitignore_txt,
|
| 381 |
"tiny_computer_agent_app.py": _notebook.read_text(encoding="utf-8"),
|
| 382 |
}
|
|
|
|
|
|
|
|
|
|
| 383 |
_missing = [f for f in ("tiny_computer_agent", "tests") if not (_root / f).is_dir()]
|
| 384 |
if _missing:
|
| 385 |
_status = mo.md(f"Nothing to publish: missing {_missing} next to `{_root}`").callout(kind="danger")
|
|
@@ -393,7 +413,7 @@ def publish_status(
|
|
| 393 |
repo_type="model",
|
| 394 |
commit_message=f"Publish {_name} from marimo",
|
| 395 |
)
|
| 396 |
-
for _folder in
|
| 397 |
_api.upload_folder(
|
| 398 |
folder_path=str(_root / _folder),
|
| 399 |
path_in_repo=_folder,
|
|
@@ -403,7 +423,7 @@ def publish_status(
|
|
| 403 |
ignore_patterns=["__pycache__/*", "*.pyc"],
|
| 404 |
)
|
| 405 |
_status = mo.md(
|
| 406 |
-
f"✅ Published {len(_files)} files and
|
| 407 |
f"[{REPO_ID}](https://huggingface.co/{REPO_ID})"
|
| 408 |
).callout(kind="success")
|
| 409 |
except Exception as _e:
|
|
|
|
| 22 |
@app.cell(hide_code=True)
|
| 23 |
def readme():
|
| 24 |
readme_md = '''
|
| 25 |
+
|
| 26 |
---
|
| 27 |
title: Tiny Computer Agent
|
| 28 |
emoji: 🖱️
|
|
|
|
| 175 |
was actually drawn at, clicks it for real, and checks the app reacted. It is opt-in
|
| 176 |
because it moves your mouse — run it on a throwaway desktop.
|
| 177 |
|
| 178 |
+
### Testing on Windows (or macOS)
|
| 179 |
+
|
| 180 |
+
This repo's live test was run on Linux/X11, because that is what the build container
|
| 181 |
+
had. For real Windows coverage, mirror the repo to GitHub: `.github/workflows/live-desktop.yml`
|
| 182 |
+
runs the unit suite **and** the live desktop test on `windows-latest` — a real Windows
|
| 183 |
+
desktop session, free for public repositories — plus the same test under Xvfb on Linux,
|
| 184 |
+
and a preflight on macOS. Trigger it from the Actions tab.
|
| 185 |
+
|
| 186 |
+
Nothing in the harness is Linux-specific: pyautogui uses `SendInput` on Windows, and
|
| 187 |
+
`GrabScreen` already falls back to Pillow's `ImageGrab` there. Two things can only fail
|
| 188 |
+
on a real Windows desktop, which is exactly why the job exists: **DPI scaling** (that is
|
| 189 |
+
what `--scale` is for) and the native input backend. At 125% or 150% display scaling,
|
| 190 |
+
expect the coordinate assertion in `test_live_desktop.py` to be the first thing that
|
| 191 |
+
fails — that is the signal, not a nuisance.
|
| 192 |
+
|
| 193 |
## How it works
|
| 194 |
|
| 195 |
| Module | Job |
|
|
|
|
| 250 |
- [x] A real click on a real window, with a verified outcome
|
| 251 |
- [x] Blind-grid-click guard, echo detection, no-progress guard
|
| 252 |
- [ ] Make the VLM reliable on a live screen (it echoes the prompt instead of choosing)
|
| 253 |
+
- [ ] Windows run of the live test (workflow included; needs a GitHub mirror)
|
| 254 |
- [ ] Accessibility-tree detector (AT-SPI on Linux, UI Automation on Windows)
|
| 255 |
- [ ] Score episodes: replay a trace and count how often the choice was right
|
| 256 |
- [ ] marimo console: live screen preview, action log, pause / resume / step
|
|
|
|
| 397 |
".gitignore": gitignore_txt,
|
| 398 |
"tiny_computer_agent_app.py": _notebook.read_text(encoding="utf-8"),
|
| 399 |
}
|
| 400 |
+
_folders = ["tiny_computer_agent", "tests"]
|
| 401 |
+
if (_root / ".github").is_dir():
|
| 402 |
+
_folders.append(".github") # the CI that tests on real Windows
|
| 403 |
_missing = [f for f in ("tiny_computer_agent", "tests") if not (_root / f).is_dir()]
|
| 404 |
if _missing:
|
| 405 |
_status = mo.md(f"Nothing to publish: missing {_missing} next to `{_root}`").callout(kind="danger")
|
|
|
|
| 413 |
repo_type="model",
|
| 414 |
commit_message=f"Publish {_name} from marimo",
|
| 415 |
)
|
| 416 |
+
for _folder in _folders:
|
| 417 |
_api.upload_folder(
|
| 418 |
folder_path=str(_root / _folder),
|
| 419 |
path_in_repo=_folder,
|
|
|
|
| 423 |
ignore_patterns=["__pycache__/*", "*.pyc"],
|
| 424 |
)
|
| 425 |
_status = mo.md(
|
| 426 |
+
f"✅ Published {len(_files)} files and {len(_folders)} folders to "
|
| 427 |
f"[{REPO_ID}](https://huggingface.co/{REPO_ID})"
|
| 428 |
).callout(kind="success")
|
| 429 |
except Exception as _e:
|