examples/run_benchmark.py

"""
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
FILE: run_benchmark.py
FOLDER: examples/
PURPOSE: Runs 10 episodes with 3 review types and prints comparison stats
USED BY: Judges evaluating the reward heuristic dynamically
KEY CLASSES/FUNCTIONS: main()
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
"""

import os
import asyncio
from codereview_env.client import CodeReviewEnv
from codereview_env.models import CodeReviewAction

TYPES = [
    {"name": "Generic (bad)", "text": "LGTM looks good!"},
    {"name": "Medium", "text": "There might be an issue here. Consider fixing it."},
    {
        "name": "Specific (good)",
        "text": "Line 3: Critical bug — The indexing is exceeding array bounds causing a runtime error. Switch `>` to `>=` to patch it safely.",
    },
]


async def main():
    print("============================================================")
    print(" 📊 CodeReview-Env Dynamic Benchmarking ")
    print(" Running 10 episodes for each review type.")
    print("============================================================\n")

    port = os.getenv("PORT", "8000")
    base_url = f"http://localhost:{port}"

    results = {"Generic (bad)": [], "Medium": [], "Specific (good)": []}

    async with CodeReviewEnv(base_url=base_url) as env:
        for t in TYPES:
            print(f"Evaluating: {t['name']}...")
            for i in range(10):
                try:
                    await env.reset()
                    action = CodeReviewAction(
                        review_comment=t["text"], severity="major"
                    )
                    result = await env.step(action)
                    rew = float(
                        result.reward
                        if hasattr(result, "reward")
                        else (
                            result.get("reward", 0.05) if hasattr(result, "get") else 0.05
                        )
                    )
                    results[t["name"]].append(rew)
                except Exception as e:
                    print(
                        f"Error on iteration {i}: {e}. Ensure API relies on localhost:{port}"
                    )
                    break

    # Calculate statistics
    stats = {}
    for k, v in results.items():
        if len(v) == 0:
            stats[k] = (0.05, 0.05, 0.05)
            continue
        stats[k] = (sum(v) / len(v), min(v), max(v))  # Avg  # Min  # Max

    # Output Table
    print("\n┌────────────────┬──────────────┬──────────────┬──────────────┐")
    print("│ Review Type    │ Avg Reward   │ Min Reward   │ Max Reward   │")
    print("├────────────────┼──────────────┼──────────────┼──────────────┤")
    for k, (avg, mn, mx) in stats.items():
        print(f"│ {k:<14} │    {avg:.2f}      │    {mn:.2f}      │    {mx:.2f}      │")
    print("└────────────────┴──────────────┴──────────────┴──────────────┘\n")


if __name__ == "__main__":
    asyncio.run(main())