src/react_agent/benchmark.py

import asyncio
import time
from rich.console import Console
from rich.table import Table
from rich.panel import Panel

from src import retriever
from src import generator
from src import react_agent
from src.retriever import client

console = Console()

TEST_QUERIES = [
    "What is the punishment for robbery?",
    "What if the robber is a minor?",
    "Can you repeat the rule for the maximum detention time?"
]

async def run_benchmark():
    console.print(Panel(
        "[bold yellow]Loading indices for benchmark...[/bold yellow]",
        border_style="yellow"
    ))
    retriever.load("tree")
    
    # 1. Deterministic Pipeline Benchmark
    console.print("\n[bold green]=== RUNNING DETERMINISTIC PIPELINE ===[/bold green]")
    det_history = []
    det_last_retrieval = None
    det_results = []
    
    for q in TEST_QUERIES:
        client.new_calls_count = 0
        start = time.time()
        res = await generator.generate(q, det_history, det_last_retrieval)
        elapsed = round((time.time() - start) * 1000)
        
        det_history.append({"user": q, "assistant": res["answer"]})
        det_last_retrieval = res["retrieval"]
        
        det_results.append({
            "query": q,
            "latency": elapsed,
            "calls": client.new_calls_count,
            "citations": len(res["citations"]),
            "confidence": res["confidence"]
        })
        console.print(f"  - Query: '[dim]{q}[/dim]' -> [green]Done[/green] in {elapsed}ms | LLM Calls: {client.new_calls_count}")
        
    # 2. ReAct Agent Pipeline Benchmark
    console.print("\n[bold magenta]=== RUNNING REACT AGENT PIPELINE ===[/bold magenta]")
    react_history = []
    react_results = []
    
    for q in TEST_QUERIES:
        client.new_calls_count = 0
        start = time.time()
        res = await react_agent.generate(q, react_history)
        elapsed = round((time.time() - start) * 1000)
        
        react_history.append({"user": q, "assistant": res["answer"]})
        
        react_results.append({
            "query": q,
            "latency": elapsed,
            "calls": client.new_calls_count,
            "citations": len(res["citations"]),
            "confidence": res["confidence"]
        })
        console.print(f"  - Query: '[dim]{q}[/dim]' -> [magenta]Done[/magenta] in {elapsed}ms | LLM Calls: {client.new_calls_count}")

        
    # 3. Render Comparison Table
    table = Table(
        title="[bold white]Benchmark Comparison: Deterministic vs ReAct[/bold white]",
        show_header=True,
        header_style="bold cyan",
        border_style="dim white"
    )
    table.add_column("Query Scenario", style="cyan", width=35)
    table.add_column("Det Latency", style="green", justify="right")
    table.add_column("ReAct Latency", style="magenta", justify="right")
    table.add_column("Det Calls", style="green", justify="center")
    table.add_column("ReAct Calls", style="magenta", justify="center")
    table.add_column("Det Citations", style="green", justify="center")
    table.add_column("ReAct Citations", style="magenta", justify="center")
    
    for idx, q in enumerate(TEST_QUERIES):
        d = det_results[idx]
        r = react_results[idx]
        table.add_row(
            q,
            f"{d['latency']} ms",
            f"{r['latency']} ms",
            str(d['calls']),
            str(r['calls']),
            str(d['citations']),
            str(r['citations'])
        )
        
    console.print("\n")
    console.print(table)
    console.print()

if __name__ == "__main__":
    asyncio.run(run_benchmark())