scripts/run_test.py

"""Headless end-to-end test: idea -> discovery -> engineering -> review.



Answers discovery questions automatically (no stdin) so the whole workflow can

run unattended against the real Cursor provider. Prints a per-agent cost table

(duration + estimated tokens) read from the ExecutionTracker.



Run:  python -m scripts.run_test "YOUR BUSINESS IDEA"

"""

from __future__ import annotations

import asyncio
import sys

from agentic_core.artifacts import ArtifactStore, render_all
from agentic_core.config import get_settings
from agentic_core.llm import LLMService, create_llm_provider
from agentic_core.orchestrator import DiscoveryError, EventBus, ExecutionTracker, Orchestrator
from agentic_core.project_store import ProjectStore

MAX_DISCOVERY_ROUNDS = 8


def auto_answer(question) -> str:
    """Answer any discovery question: pick the first option when available,

    otherwise fall back to a definitive, concrete reply that stays generic so a

    benchmark run measures the idea actually passed in, never a hard-coded one."""
    if getattr(question, "options", None):
        return question.options[0]
    return (
        "v1 ships as a responsive web app that works on mobile and desktop browsers. "
        "Target users and their roles follow the business idea I gave you. Include "
        "only the features needed for the idea to work in its first version, keep "
        "authentication simple (email + password), and prefer a single deployment "
        "with standard monitoring. Record anything genuinely unspecified as an "
        "assumption rather than inventing requirements."
    )


async def run(idea: str) -> None:
    settings = get_settings()
    provider = create_llm_provider(settings)
    llm_service = LLMService(provider, settings)
    event_bus = EventBus()
    tracker = ExecutionTracker(settings.runs_dir)
    orchestrator = Orchestrator(llm_service, event_bus, tracker, settings)
    project_store = ProjectStore(settings.db_path, legacy_dir=settings.projects_dir)

    context = project_store.create(idea)
    print(f"[project {context.project_id}] {idea}\n")

    output = await orchestrator.discovery_turn(context, idea)
    rounds = 1
    while output.status != "ready" and rounds < MAX_DISCOVERY_ROUNDS:
        if not output.questions:
            # Agent needs more but asked nothing; nudge it to proceed or go ready.
            context.add_turn("user", "Please continue.")
            output = await orchestrator.discovery_turn(context)
            rounds += 1
            continue
        print(f"[discovery round {rounds}] {len(output.questions)} question(s):")
        for q in output.questions:
            print(f"  - {q.question}")
            context.add_turn("user", auto_answer(q))
        output = await orchestrator.discovery_turn(context)
        rounds += 1

    if output.status != "ready":
        print("Discovery did not reach 'ready'; aborting.")
        return

    print(f"\nDiscovered after {rounds} round(s). Summary: {output.summary}\n")

    orchestrator.confirm(context)
    results = await orchestrator.generate(context)
    project_store.save(context)

    if context.status in ("approved", "revised"):
        files = render_all(context)
        artifact_store = ArtifactStore(settings.artifacts_dir)
        for name, content in files.items():
            artifact_store.write(context.project_id, name, content)
        print(f"\nArtifacts ({len(files)}): {settings.artifacts_dir / context.project_id}")
        for name in sorted(files):
            print(f"  - {name}")
    else:
        print(f"\nWorkflow finished with status: {context.status}")

    _print_call_summary(results)
    _print_summary(tracker, context.project_id)


def _print_call_summary(results: dict) -> None:
    counts = results.get("call_counts", {})
    revisions = results.get("revisions", {})
    if not counts:
        return
    order = ["requirements", "architecture", "database", "api", "devops", "reviewer"]
    print("\n" + "=" * 78)
    print("LLM CALLS (per agent)")
    print("=" * 78)
    total = 0
    for agent in order:
        n = counts.get(agent, 0)
        total += n
        revision = f" (revised x{revisions.get(agent, 0)})" if revisions.get(agent, 0) else ""
        print(f"  {agent:<14} {n}{revision}")
    print(f"  {'TOTAL':<14} {total}")


def _print_summary(tracker, project_id: str) -> None:
    records = tracker.list(project_id)
    if not records:
        print("\nNo tracked runs found for this project.")
        return

    # Every agent run writes two tracker records (status "started", then the
    # completed record). Only completed records represent actual provider calls.
    rows = [r for r in records if r.status != "started"]

    by_agent: dict[str, list] = {}
    for r in rows:
        by_agent.setdefault(r.agent, []).append(r)

    print("\n" + "=" * 132)
    print(f"{'agent':<14}{'status':<10}{'ms':>8}{'ttft s':>8}{'in tok':>10}{'out tok':>10}{'schema tok':>11}{'repairs':>8}{'calls':>6}  {'model':<22}")
    print("-" * 132)
    total_ms = total_in = total_out = total_schema = total_repairs = 0
    total_calls = 0
    slowest = ("", 0)
    largest_output = ("", 0)
    largest_prompt = ("", 0)
    for r in rows:
        ms = r.duration_ms or 0
        t_in = r.input_tokens or (r.input_chars // 4)
        t_out = r.output_tokens or (r.output_chars // 4)
        t_schema = r.schema_chars // 4
        repairs = r.retry_count or 0
        total_ms += ms
        total_in += t_in
        total_out += t_out
        total_schema += t_schema
        total_repairs += repairs
        if ms > slowest[1]:
            slowest = (r.agent, ms)
        if t_out > largest_output[1]:
            largest_output = (r.agent, t_out)
        if t_in > largest_prompt[1]:
            largest_prompt = (r.agent, t_in)
    for agent, agent_rows in by_agent.items():
        calls = len(agent_rows)
        total_calls += calls
        ms = sum(r.duration_ms or 0 for r in agent_rows)
        t_in = sum((r.input_tokens or (r.input_chars // 4)) for r in agent_rows)
        t_out = sum((r.output_tokens or (r.output_chars // 4)) for r in agent_rows)
        t_schema = sum((r.schema_chars // 4) for r in agent_rows)
        repairs = sum(r.retry_count or 0 for r in agent_rows)
        last = agent_rows[-1]
        print(f"{agent:<14}{last.status:<10}{ms:>8}{last.ttft_s or 0.0:>8.1f}{t_in:>10,}{t_out:>10,}{t_schema:>11,}{repairs:>8}{calls:>6}  {(last.model or '')[:22]:<22}")
    print("-" * 132)
    print(f"{'TOTAL':<14}{'':<10}{total_ms:>8}{'':>8}{total_in:>10,}{total_out:>10,}{total_schema:>11,}{total_repairs:>8}{total_calls:>6}")

    # Real provider calls: each completed record is one agent run; each run makes
    # 1 + (structured-output repairs) provider round-trips. Repairs happen inside
    # LLMService and are not separate records, so they must be added on top.
    real_provider_calls = total_calls + total_repairs
    discovery_rows = [r for r in rows if r.agent == "discovery"]
    engineering_rows = [r for r in rows if r.agent != "discovery"]
    discovery_calls = len(discovery_rows)
    discovery_repairs = sum(r.retry_count or 0 for r in discovery_rows)
    engineering_ms = sum(r.duration_ms or 0 for r in engineering_rows)
    engineering_calls = len(engineering_rows)
    engineering_repairs = sum(r.retry_count or 0 for r in engineering_rows)

    print(f"\nDiscovery runs: {discovery_calls}  (repairs: {discovery_repairs})")
    print(f"Engineering + review runs: {engineering_calls}  (repairs: {engineering_repairs})")
    print(f"Real provider calls (runs + internal repairs): ~{real_provider_calls}")
    print(f"Engineering wall-clock (requirements..review): {engineering_ms / 1000:.1f}s")
    print(f"Total wall-clock (incl. discovery):            {total_ms / 1000:.1f}s")
    print(f"Average agent latency: {total_ms / max(len(rows), 1) / 1000:.1f}s")
    print(f"Slowest agent: {slowest[0]} ({slowest[1] / 1000:.1f}s)")
    print(f"Largest prompt input: {largest_prompt[0]} ({largest_prompt[1]:,} est tokens)")
    print(f"Largest output: {largest_output[0]} ({largest_output[1]:,} est tokens)")
    reviewer = [r for r in engineering_rows if r.agent == "reviewer"]
    if reviewer:
        print(f"Reviewer prompt input: {reviewer[-1].input_chars // 4:,} est tokens")
    print("\nNote: estimated input tokens are total prompt chars sent for the agent,")
    print("which includes repair resends for any agent that needed a JSON repair.")

    print("\n" + "=" * 132)
    print("TOKEN ACCOUNTING")
    print("=" * 132)
    print(f"Estimated application-visible tokens (chars/4): ~{total_in + total_out:,}")
    print(f"  - input (prompts incl. embedded schema): ~{total_in:,}")
    print(f"  - output (model responses):             ~{total_out:,}")
    print(f"  - embedded JSON schema:                  ~{total_schema:,} of the input")
    print("Provider-reported usage: NOT exposed by the Cursor Cloud Agents API.")
    print("  The Cursor dashboard counts framework, tooling and reasoning tokens")
    print("  that our provider call cannot observe; it is NOT comparable 1:1 with")
    print("  the estimated application-visible values above.")


if __name__ == "__main__":
    idea = " ".join(sys.argv[1:]) or "A marketplace connecting dog groomers with pet owners for bookings, reminders, and online payment."
    try:
        asyncio.run(run(idea))
    except KeyboardInterrupt:
        print("\nBye.")