Publish tiny_computer_agent_app.py from marimo

tiny_computer_agent_app.py

@@ -0,0 +1,408 @@
+import marimo
+
+__generated_with = "0.25.2"
+app = marimo.App(width="medium", auto_download=["html"])
+
+with app.setup:
+    import os
+    from pathlib import Path
+
+    import marimo as mo
+    from huggingface_hub import HfApi
+
+
+
+@app.cell(hide_code=True)
+def readme():
+    readme_md = '''
+    ---
+    title: Tiny Computer Agent
+    emoji: 🖱️
+    colorFrom: indigo
+    colorTo: green
+    license: mit
+    tags:
+      - computer-use
+      - gui-agent
+      - agentic-harness
+      - local-first
+      - tiny-models
+      - offline
+      - marimo
+    ---
+
+    # 🖱️ Tiny Computer Agent
+
+    **An agentic harness that drives a real computer with a tiny, fully local model.**
+    No cloud API, no API keys, no telemetry — and a core that imports nothing outside
+    the Python standard library.
+
+    > **Status: early alpha.** The loop runs end to end today, with a scripted screen,
+    > dry-run execution, a CLI and 56 offline tests. The vision backend is a
+    > documented seam, not a feature — see [what works](#-what-works-today).
+
+    ## The idea, in one loop
+
+    ```
+        observe         think          act           verify
+      +---------+    +---------+    +---------+    +---------+
+      | screen  |--->|  tiny   |--->| mouse / |--->|  next   |
+      |  frame  |    |  model  |    | keyboard|    |  frame  |
+      +---------+    +---------+    +---------+    +----+----+
+                                                        |
+      <-------------------------------------------------+
+            repeat until done, or the step budget runs out
+    ```
+
+    1. **Observe** — grab the screen, downscale it, optionally attach OCR text or the
+       accessibility tree so the model never has to guess where things are.
+    2. **Think** — a tiny (≤ 3B, quantised) vision-language model emits **exactly one
+       action** from a closed vocabulary.
+    3. **Act** — the harness executes it with OS-level input (mouse, keyboard, scroll).
+    4. **Verify** — the next frame shows whether it worked. Repeat, bounded by a step
+       budget and a kill switch.
+
+    ## What works today
+
+    ```
+    $ python -m tiny_computer_agent
+    goal: click 'Save' to save the document
+    mode: dry-run  |  screen: scripted demo  |  kill switch: create /tmp/tiny-computer-agent.STOP to stop
+
+      step 0 | wait seconds=0.5 reason="'Save' is not on screen yet" | ok: (dry-run) would wait 0.5s
+      step 1 | click label='Save' reason="matched 'Save' in the element index" | ok: (dry-run) would click -> pixel (360, 132)
+      step 2 | done reason="already clicked 'Save'; goal assumed met" | ok: goal reported met
+
+    result: success - brain reported done: already clicked 'Save'; goal assumed met
+    ```
+
+    Shipped and tested:
+
+    - **the loop** — `AgentLoop` runs observe → think → act → verify, **re-reads the
+      screen after every action**, and stops on done, on a step budget, on the kill
+      switch, on a policy refusal, on a failed action, or on a brain error. Every step
+      is recorded in an `Episode` you can render or turn into rows.
+    - **a closed action vocabulary** — `click` `type` `key` `scroll` `wait` `done`,
+      validated *before* anything executes. A model that invents a verb gets a recorded
+      error, not a click.
+    - **three brains** — `MockBrain` replays a script; `PlannerBrain` is rule-based and
+      genuinely clicks the element matching the goal (it waits for a dialog instead of
+      flailing, and refuses to click the same label twice); `VLMBrain` is the seam —
+      prompt assembly and reply validation are done, `_generate()` is yours.
+    - **two executors** — `DryRunExecutor` is the default: it logs the plan and resolves
+      every label to a pixel. `PyAutoGuiExecutor` does real input and **refuses to click
+      when a label matches no element**, because a tiny model guessing pixels is how you
+      click "Delete account". Dry-run fails exactly where live input refuses, so a
+      rehearsal never gives false confidence.
+    - **policy gates** — deny-listed text (`sudo`, `rm -rf`, "password", "api_key"),
+      deny-listed widgets (terminal, keychain, password manager, wallet, bank), four
+      deny-listed key chords, a kill-switch file checked before every action, and a step
+      budget.
+    - **real input needs two deliberate flags** — `--real-input` *and*
+      `TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1`. Default is a dry run.
+    - **56 tests** needing no display, no model and no network.
+
+    What does **not** work yet: seeing a real screen *usefully*. `GrabScreen` captures
+    pixels through `mss`, but nothing yet turns pixels into an element index (OCR or
+    AT-SPI), so a live screen currently yields zero elements — which is exactly what the
+    label path depends on. And no model is wired up.
+
+    ## Install and run
+
+    ```bash
+    pip install -e .                          # core: standard library only, no deps
+    pytest                                    # 56 offline tests
+    python -m tiny_computer_agent             # scripted demo, dry run
+    python -m tiny_computer_agent --live      # your real screen, still dry run
+    ```
+
+    ## Put it on your own task
+
+    ```python
+    from tiny_computer_agent import AgentLoop, Element, FakeScreen, MockBrain
+
+    screen = FakeScreen([(Element("Save", x=320, y=120, width=80, height=24),)])
+    brain = MockBrain([{"kind": "click", "label": "Save"},
+                       {"kind": "done", "reason": "saved"}])
+
+    episode = AgentLoop(screen=screen, brain=brain).run("save the document")
+    print(episode.render())
+    episode.to_rows()          # rows for a dataframe or notebook table
+    ```
+
+    Swap `FakeScreen` for `GrabScreen()` and `MockBrain` for your `VLMBrain` subclass and
+    that is the whole distance between the tests and a live agent.
+
+    ## Why "tiny" is the whole point
+
+    The harness does the heavy lifting, so the model does not have to be clever:
+
+    | Job | Who does it |
+    |---|---|
+    | Finding *where* a button is | OCR / accessibility tree / element index — not the model |
+    | Choosing *what* to do | a tiny VLM over a **closed action menu** (click, type, key, scroll, wait, done) |
+    | Knowing whether it worked | the next frame, plus harness-side checks |
+    | Recovering from a bad step | the retry policy in `loop.py`, not the weights |
+
+    That keeps the agent offline, fast to start, and small enough that a laptop GPU
+    (or plain CPU) can run it on the same box it is controlling.
+
+    ## Candidate backends
+
+    Pluggable behind one interface; nothing here needs a network call at run time.
+
+    - **Moondream 2** / **SmolVLM-256M / 500M** — a few hundred MB, safe default
+    - **Qwen2-VL-2B-Instruct**, **Florence-2-base** — stronger grounding, still tiny
+    - **OCR + text-only planner** — no VLM at all; the cheapest credible baseline
+    - **mock brain** — deterministic scripted actions, used to test the harness itself
+
+    ## Layout
+
+    ```
+    tiny-computer-agent/
+    ├── README.md
+    ├── LICENSE
+    ├── pyproject.toml
+    ├── tiny_computer_agent_app.py   # marimo console: run an episode, publish this card
+    ├── tiny_computer_agent/
+    │   ├── actions.py    # the closed action vocabulary
+    │   ├── observe.py    # Observation + element index; FakeScreen, GrabScreen
+    │   ├── brain.py      # MockBrain, PlannerBrain, VLMBrain (the seam)
+    │   ├── act.py        # DryRunExecutor (default) + PyAutoGuiExecutor
+    │   ├── safety.py     # Policy: deny lists, kill switch, step budget
+    │   ├── loop.py       # AgentLoop, Episode, Step
+    │   └── __main__.py   # CLI
+    └── tests/            # 56 tests: no display, no model, no network
+    ```
+
+    ## Safety, because this thing has your mouse
+
+    - Run it in a **VM, container, or throwaway desktop account** — not your real session.
+    - Dry run is the default; read the whole plan before letting it near your pointer.
+    - Deny-list for shells, terminals, payment flows and password managers.
+    - A step budget plus a kill-switch file checked before every action.
+    - **Local-first, for real:** screen pixels and keystrokes never leave the machine.
+
+    ## Roadmap
+
+    - [x] `loop.py` + `act.py` with the mock brain — the harness is testable before any model download
+    - [x] policy gates: deny lists, kill switch, step budget, dry run by default
+    - [ ] OCR / AT-SPI element index, so a live screen has clickable labels
+    - [ ] tiny-VLM grounding backend, behind `VLMBrain._generate()`
+    - [ ] marimo console: live screen preview, action log, pause / resume / step
+    - [ ] persist episodes, so failed runs can be replayed and scored
+
+    ## License
+
+    MIT — see [LICENSE](LICENSE).
+    '''
+
+    mo.md(readme_md)
+
+    return (readme_md,)
+
+
+@app.cell(hide_code=True)
+def repo_files():
+    pyproject_toml = '''
+    [build-system]
+    requires = ["setuptools>=68"]
+    build-backend = "setuptools.build_meta"
+
+    [project]
+    name = "tiny-computer-agent"
+    version = "0.1.0"
+    description = "An agentic harness that drives a real computer with a tiny, fully local model - no cloud API."
+    readme = "README.md"
+    requires-python = ">=3.10"
+    license = { text = "MIT" }
+    authors = [{ name = "hkblockchaintechnology" }]
+    keywords = ["computer-use", "gui-agent", "agentic-harness", "local-first", "tiny-models"]
+    classifiers = [
+      "Development Status :: 3 - Alpha",
+      "Intended Audience :: Developers",
+      "Programming Language :: Python :: 3 :: Only",
+      "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    ]
+    dependencies = []   # the harness core imports nothing outside the standard library
+
+    [project.optional-dependencies]
+    desktop = ["mss>=9.0", "pyautogui>=0.9", "pillow>=10"]
+    vision = ["transformers>=4.48", "torch>=2.2", "pillow>=10"]
+    dev = ["pytest>=8"]
+
+    [project.scripts]
+    tiny-computer-agent = "tiny_computer_agent.__main__:main"
+
+    [project.urls]
+    Homepage = "https://huggingface.co/hkblockchaintechnology/tiny-computer-agent"
+
+    [tool.setuptools]
+    packages = ["tiny_computer_agent"]
+
+    [tool.pytest.ini_options]
+    testpaths = ["tests"]
+    '''
+
+    license_txt = '''MIT License
+
+    Copyright (c) 2026 hkblockchaintechnology
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+    '''
+
+    gitignore_txt = '''__pycache__/
+    *.py[cod]
+    .venv/
+    venv/
+    .env
+    *.gguf
+    models/
+    runs/
+    traces/
+    .ipynb_checkpoints/
+    .DS_Store
+    '''
+
+    mo.md(f"{len(pyproject_toml) + len(license_txt) + len(gitignore_txt)} chars of packaging ready to publish")
+
+    return gitignore_txt, license_txt, pyproject_toml
+
+
+@app.cell
+def publish():
+    REPO_ID = "hkblockchaintechnology/tiny-computer-agent"
+
+    publish_button = mo.ui.button(
+        label="⬆️ Publish README.md to the Hub",
+        value=0,
+        on_click=lambda value: value + 1,
+    )
+
+    publish_button
+
+    return REPO_ID, publish_button
+
+
+@app.cell
+def publish_status(
+    REPO_ID,
+    gitignore_txt,
+    license_txt,
+    publish_button,
+    pyproject_toml,
+    readme_md,
+):
+    _status = mo.md(
+        "This notebook owns the project: the card, the packaging files, the package and "
+        "the tests. Click **Publish** to push all of it to the Hub."
+    ).callout(kind="info")
+
+    if publish_button.value:
+        _token = os.environ.get("HF_TOKEN")
+        if not _token:
+            _status = mo.md(
+                "No `HF_TOKEN` in the environment. Run `export HF_TOKEN=...` "
+                "(or `hf auth login`), then restart the kernel."
+            ).callout(kind="warn")
+        else:
+            _api = HfApi(token=_token)
+            _root = Path(__file__).parent
+            _notebook = Path(__file__)
+            _files = {
+                "README.md": readme_md,
+                "pyproject.toml": pyproject_toml,
+                "LICENSE": license_txt,
+                ".gitignore": gitignore_txt,
+                "tiny_computer_agent_app.py": _notebook.read_text(encoding="utf-8"),
+            }
+            _missing = [f for f in ("tiny_computer_agent", "tests") if not (_root / f).is_dir()]
+            if _missing:
+                _status = mo.md(f"Nothing to publish: missing {_missing} next to `{_root}`").callout(kind="danger")
+            else:
+                try:
+                    for _name, _text in _files.items():
+                        _api.upload_file(
+                            path_or_fileobj=_text.encode("utf-8"),
+                            path_in_repo=_name,
+                            repo_id=REPO_ID,
+                            repo_type="model",
+                            commit_message=f"Publish {_name} from marimo",
+                        )
+                    for _folder in ("tiny_computer_agent", "tests"):
+                        _api.upload_folder(
+                            folder_path=str(_root / _folder),
+                            path_in_repo=_folder,
+                            repo_id=REPO_ID,
+                            repo_type="model",
+                            commit_message=f"Publish {_folder}/ from marimo",
+                            ignore_patterns=["__pycache__/*", "*.pyc"],
+                        )
+                    _status = mo.md(
+                        f"✅ Published {len(_files)} files and 2 folders to "
+                        f"[{REPO_ID}](https://huggingface.co/{REPO_ID})"
+                    ).callout(kind="success")
+                except Exception as _e:
+                    _status = mo.md(f"Publish failed: `{type(_e).__name__}: {_e}`").callout(kind="danger")
+
+    _status
+
+    return
+
+
+@app.cell
+def demo():
+    from tiny_computer_agent import (
+        AgentLoop, DryRunExecutor, Element, FakeScreen, PlannerBrain, Policy,
+    )
+
+    _scripted_screen = FakeScreen(
+        [
+            (Element("Terminal", x=0, y=0, width=40, height=24),
+             Element("Files", x=60, y=0, width=40, height=24)),
+            (Element("Name", kind="input", x=100, y=120, width=200, height=24),
+             Element("Save", x=320, y=120, width=80, height=24)),
+            (Element("Save", x=320, y=120, width=80, height=24),),
+        ],
+        text="Untitled document - unsaved changes",
+    )
+
+    demo_episode = AgentLoop(
+        screen=_scripted_screen,
+        brain=PlannerBrain(),
+        executor=DryRunExecutor(),
+        policy=Policy(step_budget=6),
+    ).run("click \'Save\' to save the document")
+
+    mo.vstack(
+        [
+            mo.md(
+                "**Scripted episode, dry run** — no model, no display, nothing touched. "
+                "The planner waits for the dialog, clicks the label, and stops."
+            ),
+            mo.ui.table(demo_episode.to_rows(), selection=None),
+            mo.md(f"`{demo_episode.render().splitlines()[1]}`"),
+        ]
+    )
+
+    return
+
+
+if __name__ == "__main__":
+    app.run()