"""Ground truth for the duel: where the target element really is on screen. Every GTK app on the virtual desktop exposes its widgets through the AT-SPI accessibility bus. a11y_dump.py lists the visible ones with their screen boxes, so a predicted click is scored as a hit when it lands inside the box, whatever the current theme, font size, or window layout. """ import json import re import subprocess import time from dataclasses import dataclass from pathlib import Path DUMP = Path(__file__).with_name("a11y_dump.py") @dataclass class Element: app: str role: str name: str x: int y: int w: int h: int @property def box(self): return self.x, self.y, self.x + self.w, self.y + self.h @property def center(self): return self.x + self.w // 2, self.y + self.h // 2 def contains(self, point): px, py = point return self.x <= px < self.x + self.w and self.y <= py < self.y + self.h @dataclass class Target: """What a challenge step must hit: an element name, optionally narrowed by role/app.""" name: str role: str | None = None app: str | None = None ROLE_ALIASES = {"push button": "button"} # AT-SPI renamed the role in 2.5x def _role(role: str) -> str: return ROLE_ALIASES.get(role, role) def _norm(s: str) -> str: return re.sub(r"[_.…\s]+", " ", s).strip().lower() def elements(env, apps=()) -> list[Element]: out = subprocess.run(["/usr/bin/python3", str(DUMP), *apps], env=env, capture_output=True, text=True, timeout=30).stdout return [Element(**json.loads(line)) for line in out.splitlines() if line.startswith("{")] def find(env, target: Target, width: int, height: int, wait: float = 3.0) -> Element | None: """The smallest visible on-screen element matching the target, if any. Retries for up to `wait` seconds, since dialogs and menus can appear late. """ apps = (target.app,) if target.app else () deadline = time.time() + wait while True: matches = [ e for e in elements(env, apps) if _norm(e.name) == _norm(target.name) and (target.role is None or _role(e.role) == _role(target.role)) and e.x < width and e.y < height and e.x + e.w > 0 and e.y + e.h > 0 ] if matches or time.time() > deadline: return min(matches, key=lambda e: e.w * e.h, default=None) time.sleep(0.5)