import asyncio from playwright.async_api import async_playwright import json from vision_engine import get_elements_from_screenshot from browser_engine import take_screenshot, click_element async def main(): async with async_playwright() as p: browser = await p.chromium.launch(headless=True) page = await browser.new_page() await page.goto("https://www.google.com") # Replace with ur target # 1. Take a look img_path = await take_screenshot(page) # 2. Ask Vision model where things are json_response = get_elements_from_screenshot(img_path) elements = json.loads(json_response) # 3. Decision Logic (Hardcoded for testing) target = next((e for e in elements if e["name"] == "search_magnifying_glass"), None) if target: x = (target["box"][0] + target["box"][2]) / 2 y = (target["box"][1] + target["box"][3]) / 2 await click_element(page, x, y) await browser.close() asyncio.run(main())