scripts/playwright_chat_turn.py

"""Browser harness for the LIVE chat turn (requires llama.cpp on LLAMA_CPP_PORT).

Loads a fixture, sends one message, and waits for the streamed assistant reply +
any canvas change the agent's tools produce. Captures chatbot text, node delta,
and a screenshot. This is the only harness that needs the LLM server up.

    .venv/Scripts/python.exe scripts/playwright_chat_turn.py ["prompt"]
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

from playwright.sync_api import sync_playwright

BASE = "http://127.0.0.1:8000/"
OUT = Path(__file__).resolve().parent.parent / ".playwright-out"
OUT.mkdir(exist_ok=True)
PROMPT = (
    sys.argv[1]
    if len(sys.argv) > 1
    else "Reveal the hidden concepts about overfitting and explain how they relate."
)

NODE_IDS = "() => (window.__cy ? window.__cy.nodes().map(n => n.id()) : [])"
# Gradio renders bot bubbles with data-testid="bot"; fall back to any .bot text.
BOT_TEXT = """
() => {
  const els = [...document.querySelectorAll('[data-testid=\"bot\"], .bot.message, .message.bot')];
  return els.map(e => e.innerText.trim()).filter(Boolean).join('\\n---\\n');
}
"""


def main() -> int:
    console: list[str] = []
    report: dict[str, object] = {"base": BASE, "prompt": PROMPT}
    failures: list[str] = []

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page(viewport={"width": 1400, "height": 1000})
        page.on("console", lambda m: console.append(f"{m.type}: {m.text}"))
        page.on("pageerror", lambda e: console.append(f"pageerror: {e}"))
        page.add_init_script("window.__PLAYWRIGHT_TEST = true;")

        page.goto(BASE, wait_until="domcontentloaded")
        page.wait_for_selector("#lc-canvas", timeout=30_000)
        page.get_by_role("button", name="Load graph").click()
        page.wait_for_function(
            "() => window.__cy && window.__cy.nodes().length > 0", timeout=15_000
        )
        nodes_before = page.evaluate(NODE_IDS)
        report["nodes_before"] = sorted(nodes_before)

        # Send a turn. The msg textbox carries the "Ask about the graph…" placeholder.
        box = page.get_by_placeholder("Ask about the graph", exact=False)
        box.click()
        box.fill(PROMPT)
        page.get_by_role("button", name="Send", exact=True).click()

        # The turn streams; the LLM at 64K ctx may take a while. Wait up to 180s for a
        # bot bubble to appear, then a short settle for streaming + canvas patch.
        bot_text = ""
        try:
            page.wait_for_function(
                '() => { const e=[...document.querySelectorAll(\'[data-testid=\\"bot\\"], '
                ".bot.message, .message.bot')]; return e.some(x => x.innerText.trim().length > 0); }",
                timeout=180_000,
            )
        except Exception as exc:  # noqa: BLE001
            failures.append(f"no assistant reply within 180s: {exc}")
        page.wait_for_timeout(4_000)
        bot_text = page.evaluate(BOT_TEXT)

        report["assistant_reply"] = bot_text[:1500]
        report["assistant_reply_chars"] = len(bot_text)
        if not bot_text.strip():
            failures.append("assistant reply empty")

        nodes_after = page.evaluate(NODE_IDS)
        report["nodes_after"] = sorted(nodes_after)
        report["nodes_added"] = sorted(set(nodes_after) - set(nodes_before))
        report["nodes_removed"] = sorted(set(nodes_before) - set(nodes_after))

        page.screenshot(path=str(OUT / "04_chat_turn.png"), full_page=True)
        browser.close()

    report["console_tail"] = console[-25:]
    report["failures"] = failures
    report["ok"] = not failures
    (OUT / "chat_report.json").write_text(
        json.dumps(report, indent=2), encoding="utf-8"
    )
    print(json.dumps(report, indent=2))
    return 0 if not failures else 1


if __name__ == "__main__":
    sys.exit(main())