| [ |
| { |
| "id": "vocab-verbs", |
| "category": "Action vocabulary", |
| "title": "The six verbs, and only six", |
| "shows": [ |
| "ACTION_KINDS", |
| "ACTION_MENU", |
| "Action", |
| "Action.describe", |
| "Action.to_dict", |
| "parse_action" |
| ], |
| "note": "Every action is validated on construction, before anything touches the machine.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "vocab-refusals", |
| "category": "Action vocabulary", |
| "title": "Malformed actions are refused, not guessed at", |
| "shows": [ |
| "ActionError", |
| "Action", |
| "parse_action" |
| ], |
| "note": "An unknown verb or a missing target raises before the machine is touched.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "vocab-menu", |
| "category": "Action vocabulary", |
| "title": "The contract handed to a model", |
| "shows": [ |
| "ACTION_MENU" |
| ], |
| "note": "The prompt-side menu: one JSON object, six shapes, no invented verbs.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "menu-numbered", |
| "category": "Decisions", |
| "title": "A short numbered menu, in reading order", |
| "shows": [ |
| "DecisionMenu", |
| "DecisionMenu.render", |
| "MenuItem", |
| "MenuItem.label" |
| ], |
| "note": "A 256M model cannot emit coordinates; it can pick a number from twelve.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "menu-lookup", |
| "category": "Decisions", |
| "title": "Menu lookup tolerates what models actually send", |
| "shows": [ |
| "DecisionMenu.item", |
| "DecisionMenu.find", |
| "DecisionMenu.mentioned_in" |
| ], |
| "note": "1, \"2\", 2.0 and \"Save.\" all resolve; a stray word resolves to nothing.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "decision-json", |
| "category": "Decisions", |
| "title": "Reading a decision out of a small model's reply", |
| "shows": [ |
| "parse_decision", |
| "extract_json_object", |
| "extract_json_objects", |
| "DecisionMenu" |
| ], |
| "note": "The LAST JSON object wins, because a model that echoes its prompt answers last.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "decision-echo-guard", |
| "category": "Decisions", |
| "title": "An echoed prompt is refused; a right answer in the wrong field is recovered", |
| "shows": [ |
| "parse_decision", |
| "BrainError", |
| "DecisionMenu.mentioned_in" |
| ], |
| "note": "Real 256M/0.5B failure modes, both caught: no blind click either way.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "decision-garbled", |
| "category": "Decisions", |
| "title": "A garbled verb with a valid label is still usable", |
| "shows": [ |
| "parse_decision", |
| "BrainError" |
| ], |
| "note": "A 256M model really does emit {\"kind\": \"file\", \"label\": \"Save\"}.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "exec-dry-run", |
| "category": "Execution", |
| "title": "Dry run resolves every click to pixels - and touches nothing", |
| "shows": [ |
| "DryRunExecutor", |
| "DryRunExecutor.execute", |
| "resolve_point", |
| "ActionResult" |
| ], |
| "note": "The default everywhere: read a whole episode before anything clicks.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "exec-refuses-unresolved", |
| "category": "Execution", |
| "title": "Real input refuses to guess a pixel for a label it cannot find", |
| "shows": [ |
| "ActionResult.summary", |
| "InputUnavailable", |
| "PyAutoGuiExecutor", |
| "PyAutoGuiExecutor.execute" |
| ], |
| "note": "This is the gate between a hallucinated label and 'Delete account'.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "loop-end-to-end", |
| "category": "The loop", |
| "title": "observe -> think -> act -> verify, traced end to end", |
| "shows": [ |
| "AgentLoop", |
| "AgentLoop.run", |
| "Episode", |
| "Episode.render", |
| "Episode.to_rows", |
| "Step", |
| "Step.line", |
| "Step.ok" |
| ], |
| "note": "One call runs a whole episode; every step is kept for audit.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "loop-budget", |
| "category": "The loop", |
| "title": "A hard step budget, so a run cannot wander", |
| "shows": [ |
| "Policy", |
| "Policy.step_budget", |
| "AgentLoop.run" |
| ], |
| "note": "The budget ends the episode and says so; nothing executes past it.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "loop-no-progress-guard", |
| "category": "The loop", |
| "title": "Repeating one action against an unchanged screen is stopped", |
| "shows": [ |
| "AgentLoop.run", |
| "Step", |
| "Step.error" |
| ], |
| "note": "A stuck agent is not a persistent one - it is stopped and told why.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "loop-kill-switch", |
| "category": "The loop", |
| "title": "A kill-switch file aborts before the next action", |
| "shows": [ |
| "Policy.kill_switch_path", |
| "Policy.should_stop", |
| "AgentLoop.run" |
| ], |
| "note": "Checked before every single action: create the file, the run stops.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "loop-survives-failures", |
| "category": "The loop", |
| "title": "A dead screen or a broken brain is an episode, not a traceback", |
| "shows": [ |
| "AgentLoop.run", |
| "Episode.stop_reason", |
| "GoalUnreachable", |
| "BrainError" |
| ], |
| "note": "Every failure path is recorded as a failed step with a reason.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "loop-verifier", |
| "category": "The loop", |
| "title": "Success is evidence, not the model's word", |
| "shows": [ |
| "AgentLoop.run", |
| "Episode.success", |
| "Observation.text" |
| ], |
| "note": "Without a verifier, a successful action that removes its own target looks like a failure.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "loop-on-step", |
| "category": "The loop", |
| "title": "Every step can be streamed out as it happens", |
| "shows": [ |
| "AgentLoop.run", |
| "Step.line", |
| "on_step" |
| ], |
| "note": "This is the hook a live console, a progress bar or a websocket uses.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "safety-deny-text", |
| "category": "Safety", |
| "title": "Typing from the deny list never reaches the keyboard", |
| "shows": [ |
| "Policy", |
| "Policy.check", |
| "PolicyViolation", |
| "DEFAULT_DENY_TEXT" |
| ], |
| "note": "Checked before execution; a refusal ends the episode and clicks nothing.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "safety-deny-label", |
| "category": "Safety", |
| "title": "Clicking a denied widget is refused, however the label is spelled", |
| "shows": [ |
| "Policy.deny_labels", |
| "Policy.check", |
| "Observation.find" |
| ], |
| "note": "Terminal, keychain, wallet, password manager, bank - matched on the label.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "safety-deny-keys", |
| "category": "Safety", |
| "title": "Key chords on the deny list are never pressed", |
| "shows": [ |
| "Policy.deny_keys", |
| "Policy.check", |
| "PolicyViolation" |
| ], |
| "note": "ctrl+alt+t, ctrl+alt+delete and ctrl+shift+escape are refused by default.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "safety-blind-grid", |
| "category": "Safety", |
| "title": "A numbered grid cell is a blind click: allowed to plan, refused for real", |
| "shows": [ |
| "Policy.allow_grid_clicks", |
| "Policy.check", |
| "GridDetector" |
| ], |
| "note": "With real input the blind guess is refused unless you pass --allow-grid.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "screen-fake", |
| "category": "Screens", |
| "title": "A scripted screen, for testing the loop with no display", |
| "shows": [ |
| "FakeScreen", |
| "FakeScreen.capture", |
| "Observation" |
| ], |
| "note": "The final frame repeats forever, so a runaway agent cannot exhaust the script.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "screen-image", |
| "category": "Screens", |
| "title": "A still image (or a sequence) replayed as a screen", |
| "shows": [ |
| "ImageScreen", |
| "ImageScreen.capture", |
| "fake_desktop" |
| ], |
| "note": "Lets the whole pipeline be exercised with no display, and a failure be replayed.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "capture-options", |
| "category": "Screens", |
| "title": "Real capture: monitors, downscaling, and HiDPI mapping", |
| "shows": [ |
| "GrabScreen", |
| "GrabScreen.capture", |
| "GrabScreen.map_to_screen", |
| "ScreenUnavailable" |
| ], |
| "note": "Frame pixel -> screen pixel is preserved, so a click cannot drift.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "element-matching", |
| "category": "Screens", |
| "title": "Labels are compared on their alphanumeric core", |
| "shows": [ |
| "Element", |
| "Element.center", |
| "Element.matches", |
| "Observation.find", |
| "normalise_label" |
| ], |
| "note": "OCR loses spaces, models add punctuation - so nothing matches by exact string.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "observation-summary", |
| "category": "Screens", |
| "title": "The compact text a model is shown", |
| "shows": [ |
| "Observation.summary" |
| ], |
| "note": "This is the whole world as the brain sees it - capped, and honest when empty.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "detector-available", |
| "category": "Perception", |
| "title": "Probing for a detector, and building one by name", |
| "shows": [ |
| "detector_available", |
| "make_detector", |
| "DetectorUnavailable", |
| "Detector" |
| ], |
| "note": "The doctor and the CLI both ask this before promising anything.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "ocr-labels", |
| "category": "Perception", |
| "title": "OCR turns a screenshot into labelled, clickable elements", |
| "shows": [ |
| "OcrDetector", |
| "OcrDetector.detect", |
| "Element" |
| ], |
| "note": "The model never has to guess where anything is: this is the grounding.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "ocr-filters", |
| "category": "Perception", |
| "title": "Three knobs keep dialog prose out of the menu", |
| "shows": [ |
| "OcrDetector.min_score", |
| "OcrDetector.max_label_chars", |
| "OcrDetector.min_label_chars" |
| ], |
| "note": "Every extra line of noise is one more chance for a small model to click prose.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "grid-detector", |
| "category": "Perception", |
| "title": "A numbered grid, for screens with no text at all", |
| "shows": [ |
| "GridDetector", |
| "GridDetector.detect" |
| ], |
| "note": "An agent with an empty index has no legal moves; a grid always has moves.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "auto-detector", |
| "category": "Perception", |
| "title": "OCR when there is text, a grid when there is none", |
| "shows": [ |
| "AutoDetector", |
| "AutoDetector.detect", |
| "AutoDetector.coordinates_of_last_detect" |
| ], |
| "note": "The default: useful with nothing installed, better the moment OCR appears.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "indexed-screen", |
| "category": "Perception", |
| "title": "IndexedScreen: capture + detect, composed", |
| "shows": [ |
| "IndexedScreen", |
| "IndexedScreen.capture", |
| "Observation.summary" |
| ], |
| "note": "One capture() call gives the loop an index it can act on.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "display-available", |
| "category": "Preflight", |
| "title": "Is there a desktop session to drive at all?", |
| "shows": [ |
| "display_available" |
| ], |
| "note": "The first question the CLI and the doctor both ask.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "run-checks", |
| "category": "Preflight", |
| "title": "Every precondition, one list, each with the fix", |
| "shows": [ |
| "run_checks", |
| "Check", |
| "Check.render" |
| ], |
| "note": "Eight checks; a required one that fails is a blocking problem.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-doctor", |
| "category": "Preflight", |
| "title": "python -m tiny_computer_agent --doctor", |
| "shows": [ |
| "detector_available", |
| "display_available", |
| "doctor", |
| "report" |
| ], |
| "note": "The first thing to run on a new machine. Exits non-zero when not ready.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "build-agent", |
| "category": "Preflight", |
| "title": "One call wires screen + detector + brain + executor + policy", |
| "shows": [ |
| "build_agent", |
| "Policy", |
| "AgentLoop" |
| ], |
| "note": "Defaults are the safe ones: dry run, a step budget, and a kill-switch file.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "models-deps", |
| "category": "Models", |
| "title": "Which model runtime is present, and are the weights cached?", |
| "shows": [ |
| "model_dependencies", |
| "model_cached", |
| "MODEL_BRAINS", |
| "make_model_brain", |
| "DEFAULT_TEXT_MODEL", |
| "DEFAULT_VISION_MODEL", |
| "BrainError" |
| ], |
| "note": "Weights are loaded lazily, so asking is free.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "models-device", |
| "category": "Models", |
| "title": "Device and dtype selection", |
| "shows": [ |
| "resolve_device", |
| "pick_dtype" |
| ], |
| "note": "CUDA if it is there, CPU otherwise; the harness never needs a GPU.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "models-prompt", |
| "category": "Models", |
| "title": "The prompt a model actually receives - and how it learns from a dead end", |
| "shows": [ |
| "ModelBrain", |
| "ModelBrain.build_prompt", |
| "ModelBrain.decide" |
| ], |
| "note": "Placeholders, not worked examples: a weak model echoes the prompt, and an echo fails validation.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "models-prompt-history", |
| "category": "Models", |
| "title": "A second decision is told what already failed", |
| "shows": [ |
| "ModelBrain", |
| "ModelBrain.build_prompt", |
| "ModelBrain.history" |
| ], |
| "note": "'you already tried this and the screen did NOT change' is the most useful line a small model gets.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "scenario", |
| "category": "Traces & scoring", |
| "title": "A task with one right answer, and the frames to show", |
| "shows": [ |
| "Scenario", |
| "Scenario.label", |
| "Scenario.screen", |
| "Trace.render", |
| "run_scenario" |
| ], |
| "note": "Ground truth lives with the task, so a run can be scored at all.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "trace-record", |
| "category": "Traces & scoring", |
| "title": "An episode becomes a portable, pixel-free trace", |
| "shows": [ |
| "Trace", |
| "Trace.blocked", |
| "Trace.choice", |
| "Trace.decided", |
| "Trace.decision_step", |
| "Trace.frames", |
| "Trace.load", |
| "Trace.name", |
| "Trace.record", |
| "Trace.save", |
| "Trace.to_dict", |
| "Trace.to_json", |
| "TraceStep", |
| "TraceStep.observation", |
| "TraceStep.to_dict" |
| ], |
| "note": "JSON, so it diffs in review and replays on a machine with nothing installed.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "trace-replay", |
| "category": "Traces & scoring", |
| "title": "Replay a recorded trace against a different brain", |
| "shows": [ |
| "Trace.replay", |
| "Trace.observations", |
| "score_trace" |
| ], |
| "note": "The frames are the fixture; the brain is the only variable. Nothing is captured or clicked.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "score-trace", |
| "category": "Traces & scoring", |
| "title": "Counting how often the choice was right", |
| "shows": [ |
| "score_trace", |
| "score", |
| "score_replay", |
| "Verdict", |
| "Verdict.line" |
| ], |
| "note": "Scored on normalised text, so OCR spacing is forgiven - the same bargain the index makes.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "scoreboard", |
| "category": "Traces & scoring", |
| "title": "Verdicts collected into an accuracy", |
| "shows": [ |
| "Scoreboard", |
| "Scoreboard.accuracy", |
| "Scoreboard.accuracy_for", |
| "Scoreboard.add", |
| "Scoreboard.blocked", |
| "Scoreboard.brains", |
| "Scoreboard.by_brain", |
| "Scoreboard.confusion", |
| "Scoreboard.earned", |
| "Scoreboard.render", |
| "Scoreboard.to_rows", |
| "Scoreboard.total" |
| ], |
| "note": "The number the whole module exists for: which brain is right, and how often.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "benchmark", |
| "category": "Traces & scoring", |
| "title": "Every scenario through every brain, in one table", |
| "shows": [ |
| "benchmark", |
| "Scoreboard.to_rows" |
| ], |
| "note": "A fresh brain per task, so state like 'already clicked this' cannot leak between them.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "benchmark-traces", |
| "category": "Traces & scoring", |
| "title": "Replay a whole recorded suite against one brain", |
| "shows": [ |
| "benchmark_traces", |
| "load_traces", |
| "Trace.save" |
| ], |
| "note": "The literal roadmap step: the frames are fixed, nothing new is captured.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-image", |
| "category": "Command line", |
| "title": "--image example (replay a screenshot, dry run)", |
| "shows": [ |
| "main", |
| "build_parser", |
| "ImageScreen", |
| "detector_available" |
| ], |
| "note": "No display needed: the whole pipeline over a still image.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-script", |
| "category": "Command line", |
| "title": "--script (replay a fixed action list instead of a brain)", |
| "shows": [ |
| "MockBrain", |
| "demo_screen", |
| "parse_action" |
| ], |
| "note": "The script is validated exactly like a model's answer: nothing skips the gates.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-verify-text", |
| "category": "Command line", |
| "title": "--verify-text (the harness decides success, not the brain)", |
| "shows": [ |
| "AgentLoop.run", |
| "Episode.success" |
| ], |
| "note": "Without it, an action that removes its own target looks like a failure.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-trace", |
| "category": "Command line", |
| "title": "--trace / --expected (record the run, then score it)", |
| "shows": [ |
| "Trace.record", |
| "Trace.save", |
| "score_trace" |
| ], |
| "note": "Writes JSON next to the code, and prints the score straight away.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-detector", |
| "category": "Command line", |
| "title": "--no-ocr (index a numbered grid instead)", |
| "shows": [ |
| "GridDetector", |
| "make_detector", |
| "detector_available" |
| ], |
| "note": "Useful on a machine with no OCR engine, or for a canvas app.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-exit-codes", |
| "category": "Command line", |
| "title": "Exit codes: 0 success, 1 failure, 2 bad usage, 3 no screen", |
| "shows": [ |
| "main" |
| ], |
| "note": "A shell can tell 'it worked' from 'it gave up' without parsing any text.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-refusals", |
| "category": "Command line", |
| "title": "The two deliberate acts before the mouse can move", |
| "shows": [ |
| "main", |
| "ALLOW_REAL_INPUT_ENV" |
| ], |
| "note": "--real-input without the env var, and --brain vlm with no pixels: both refuse.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-live-headless", |
| "category": "Command line", |
| "title": "--live with no display is refused, with the fix", |
| "shows": [ |
| "display_available", |
| "main" |
| ], |
| "note": "It tells you what to do instead of dying inside an episode.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "cli-live-dry-run", |
| "category": "Command line", |
| "title": "--live (your real screen, dry run)", |
| "shows": [ |
| "GrabScreen", |
| "IndexedScreen", |
| "AutoDetector", |
| "main" |
| ], |
| "note": "Real capture, real OCR, a real element index - and not one click.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "desk-capture", |
| "category": "Real desktop", |
| "title": "GrabScreen: capture a real desktop", |
| "shows": [ |
| "GrabScreen", |
| "GrabScreen.capture", |
| "ScreenUnavailable" |
| ], |
| "note": "1280x800 of a real X server. mss first, Pillow's ImageGrab as the fallback.", |
| "kind": "desktop" |
| }, |
| { |
| "id": "desk-ocr-index", |
| "category": "Real desktop", |
| "title": "The element index, drawn on the pixels it came from", |
| "shows": [ |
| "OcrDetector", |
| "OcrDetector.detect", |
| "IndexedScreen", |
| "Element.center" |
| ], |
| "note": "Every box is a real OCR word with a real screen coordinate: the model picks a label, not a pixel.", |
| "kind": "desktop" |
| }, |
| { |
| "id": "desk-click", |
| "category": "Real desktop", |
| "title": "A real click, and the window's own reaction", |
| "shows": [ |
| "PyAutoGuiExecutor", |
| "AgentLoop.run", |
| "Policy.dry_run", |
| "resolve_point" |
| ], |
| "note": "Pixels in, a label chosen, a pixel resolved, a button pressed - and the dialog closed itself.", |
| "kind": "desktop" |
| }, |
| { |
| "id": "desk-verify", |
| "category": "Real desktop", |
| "title": "Success only when the harness can see it", |
| "shows": [ |
| "AgentLoop.run", |
| "Episode.success", |
| "Episode.stop_reason" |
| ], |
| "note": "The app changed its own title and wrote a marker file; the verifier saw the change.", |
| "kind": "desktop" |
| }, |
| { |
| "id": "desk-grid-fallback", |
| "category": "Real desktop", |
| "title": "A screen with no text still has legal moves", |
| "shows": [ |
| "AutoDetector", |
| "GridDetector", |
| "OcrDetector" |
| ], |
| "note": "OCR finds nothing on a canvas, so the index degrades to numbered cells - and says so.", |
| "kind": "desktop" |
| }, |
| { |
| "id": "desk-policy-refusal", |
| "category": "Real desktop", |
| "title": "The safety gate, on a real button, in a real window", |
| "shows": [ |
| "Policy.check", |
| "Policy.deny_labels", |
| "PolicyViolation", |
| "Observation.find" |
| ], |
| "note": "The Terminal button is visible and labelled - and the policy refuses to press it.", |
| "kind": "desktop" |
| }, |
| { |
| "id": "desk-cli-real-input", |
| "category": "Real desktop", |
| "title": "The same click, driven end to end from the CLI", |
| "shows": [ |
| "main", |
| "PyAutoGuiExecutor", |
| "AgentLoop.run" |
| ], |
| "note": "--live --real-input --verify-text Saved: capture, OCR, click, verify, exit code.", |
| "kind": "desktop" |
| }, |
| { |
| "id": "protocols", |
| "category": "Extensibility", |
| "title": "Every stage is a Protocol: bring your own screen, brain, detector, executor", |
| "shows": [ |
| "Brain", |
| "Brain.decide", |
| "ScreenSource", |
| "ScreenSource.capture", |
| "Detector", |
| "Detector.detect", |
| "Executor", |
| "Executor.execute", |
| "ActionResult" |
| ], |
| "note": "The loop depends on four one-method contracts, not on the shipped implementations.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "brain-mock", |
| "category": "Decisions", |
| "title": "MockBrain: replay a fixed script, exactly once", |
| "shows": [ |
| "MockBrain", |
| "MockBrain.decide", |
| "MockBrain.remaining" |
| ], |
| "note": "Deterministic and offline - the workhorse for tests, demos and replays.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "brain-planner", |
| "category": "Decisions", |
| "title": "PlannerBrain: no weights, no download, and it gives up honestly", |
| "shows": [ |
| "PlannerBrain", |
| "PlannerBrain.decide", |
| "GoalUnreachable" |
| ], |
| "note": "The proof the harness works end to end, and a real fallback when no model is wanted.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "models-swap", |
| "category": "Models", |
| "title": "The two real model backends, and lazy loading", |
| "shows": [ |
| "TextPlannerBrain", |
| "VLMBrain", |
| "TextPlannerBrain.unload", |
| "VLMBrain.unload", |
| "ModelBrain.loaded", |
| "make_model_brain" |
| ], |
| "note": "Constructing a brain costs nothing; the weights arrive on the first decide().", |
| "kind": "terminal" |
| }, |
| { |
| "id": "fake-desktop", |
| "category": "Screens", |
| "title": "The built-in synthetic desktop", |
| "shows": [ |
| "fake_desktop", |
| "write_fake_desktop", |
| "OcrDetector" |
| ], |
| "note": "Draws a screenshot-like desktop offline - the same image --image example replays.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "stepwise", |
| "category": "The loop", |
| "title": "The same loop, one turn at a time, with pause and resume", |
| "shows": [ |
| "AgentLoop", |
| "AgentLoop.begin", |
| "AgentLoop.step_once", |
| "AgentLoop.pause", |
| "AgentLoop.resume", |
| "AgentLoop.finished", |
| "AgentLoop.run", |
| "Step.line" |
| ], |
| "note": "run() is batch; a console needs the caller to own the loop. These are the same rules.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "decoding", |
| "category": "Models", |
| "title": "Constrained decoding: make an echo impossible, not merely detected", |
| "shows": [ |
| "MenuMask", |
| "digit_token_ids", |
| "choice_token_ids", |
| "pair_token_ids", |
| "read_choice", |
| "read_verdict", |
| "brevity_ladder", |
| "majority_choice", |
| "DIGITS" |
| ], |
| "note": "The 256M model's failure was answering with the prompt. A mask removes that option.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "atspi-index", |
| "category": "Perception", |
| "title": "The accessibility tree: elements the OS already knows about", |
| "shows": [ |
| "AtspiDetector", |
| "AtspiDetector.detect", |
| "AtspiDetector.text", |
| "AtspiDetector.coordinates_of_last_detect", |
| "AtspiReader", |
| "AtspiReader.read", |
| "AccessibleNode", |
| "TreeReader", |
| "TreeReader.read", |
| "accessibility_available", |
| "INTERACTIVE_ROLES", |
| "TEXT_ROLES" |
| ], |
| "note": "No pixels are read: GTK publishes every button's name and screen extents.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "models-decision-modes", |
| "category": "Models", |
| "title": "Four ways to ask a tiny model, and what each one costs", |
| "shows": [ |
| "ModelBrain", |
| "ModelBrain.loaded", |
| "TextPlannerBrain", |
| "VLMBrain", |
| "make_model_brain" |
| ], |
| "note": "json=free text, choice=one masked number, scan=yes/no per item, rank=log-odds order.", |
| "kind": "terminal" |
| }, |
| { |
| "id": "desk-atspi", |
| "category": "Real desktop", |
| "title": "Indexed by the accessibility tree, acted on with the real mouse", |
| "shows": [ |
| "AtspiDetector", |
| "AtspiDetector.detect", |
| "AtspiReader", |
| "IndexedScreen", |
| "PyAutoGuiExecutor", |
| "PlannerBrain" |
| ], |
| "note": "Roles and extents come from GTK itself; the click is a real one and the app reacts.", |
| "kind": "desktop" |
| }, |
| { |
| "id": "uia-mapping", |
| "category": "Perception", |
| "title": "The Windows backend: the same index, from UI Automation", |
| "shows": [ |
| "UiaReader", |
| "UiaReader.read", |
| "UiaDetector", |
| "UiaDetector.detect", |
| "UiaDetector.text", |
| "UiaDetector.coordinates_of_last_detect" |
| ], |
| "note": "Windows-only, so the mapping is exercised here against a synthetic control tree.", |
| "kind": "terminal" |
| } |
| ] |
