"""CLI to trigger an agent run or a regression eval and print results nicely. Usage: python scripts/run_eval.py run # run the agent on the example brief python scripts/run_eval.py run --brief brief.json python scripts/run_eval.py regression # run golden regression eval python scripts/run_eval.py status # show golden / flagged counts Uses Rich for clean terminal output. """ import argparse import json import os import sys # Make the project root importable when run as `python scripts/run_eval.py`. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from dotenv import load_dotenv # noqa: E402 load_dotenv() from rich.console import Console # noqa: E402 from rich.panel import Panel # noqa: E402 from rich.table import Table # noqa: E402 console = Console() EXAMPLE_BRIEF = { "brand": "FitFuel", "product": "High-protein meal replacement shake", "audience": "Busy professionals aged 25-40", "tone": "Energetic and no-nonsense", "goal": "Drive trial purchases", } def _load_brief(path: str | None) -> dict: if not path: return EXAMPLE_BRIEF with open(path, "r", encoding="utf-8") as f: return json.load(f) def cmd_run(args: argparse.Namespace) -> int: from agent.core import Agent brief = _load_brief(args.brief) console.print(Panel.fit(json.dumps(brief, indent=2), title="Brand Brief", border_style="cyan")) with console.status("[bold green]Running agent loop (retrieve -> generate -> evaluate -> feedback)..."): agent = Agent() result = agent.run(brief) console.print( f"\n[bold]Run #{result['run_id']}[/bold] " f"prompt=[magenta]{result['prompt_version']}[/magenta] " f"retrieved few-shot examples=[yellow]{result['retrieved_examples']}[/yellow]\n" ) table = Table(title="Generated Variants & Scores", show_lines=True) table.add_column("Variant", style="cyan", no_wrap=True) table.add_column("Copy", style="white", max_width=50) table.add_column("Hook", justify="right") table.add_column("Brand", justify="right") table.add_column("Clarity", justify="right") table.add_column("Conv", justify="right") table.add_column("Weighted", justify="right", style="bold") for o in result["outputs"]: s = o["scores"] table.add_row( o["variant_type"], o["content"], str(s["hook_strength"]), str(s["brand_alignment"]), str(s["clarity"]), str(s["conversion_intent"]), f"{s['weighted_average']:.2f}", ) console.print(table) fb = result["feedback"] console.print( f"\n[green]Promoted to golden:[/green] {fb['promoted_to_golden'] or 'none'} " f"[red]Flagged for review:[/red] {fb['flagged_for_review'] or 'none'}" ) return 0 def cmd_regression(args: argparse.Namespace) -> int: from evals.runner import format_report, run_golden_eval with console.status("[bold green]Running golden regression eval..."): report = run_golden_eval() table = Table(title=f"Regression Eval - {report.prompt_version}", show_lines=True) table.add_column("#", justify="right") table.add_column("Variant", style="cyan") table.add_column("Baseline", justify="right") table.add_column("New", justify="right") table.add_column("Delta", justify="right") table.add_column("Status", justify="center") for i, r in enumerate(report.results, 1): status = "[red]REGRESSED[/red]" if r.regressed else "[green]ok[/green]" table.add_row( str(i), r.variant_type, f"{r.baseline_score:.2f}", f"{r.new_score:.2f}", f"{r.delta:+.2f}", status, ) if report.count: console.print(table) console.print() if report.passed: console.print(Panel.fit("PASS - no entry regressed beyond tolerance.", border_style="green")) else: console.print( Panel.fit( f"FAIL - {len(report.regressions)} entry(ies) regressed. " "Do not promote this prompt version.", border_style="red", ) ) # Non-zero exit on failure so it can gate CI / a prompt swap. return 0 if report.passed else 1 def cmd_status(args: argparse.Namespace) -> int: from db.store import Store from evals.golden import GoldenDataset store = Store() golden = GoldenDataset(store=store) flagged = store.get_flagged() console.print( Panel.fit( f"Golden entries: [green]{golden.size()}[/green]\n" f"Flagged outputs: [red]{len(flagged)}[/red]", title="System Status", border_style="cyan", ) ) return 0 def main() -> int: parser = argparse.ArgumentParser(description="Self-improving ad copy agent CLI") sub = parser.add_subparsers(dest="command", required=True) p_run = sub.add_parser("run", help="Run the agent loop on a brief") p_run.add_argument("--brief", help="Path to a brief JSON file (defaults to FitFuel example)") p_run.set_defaults(func=cmd_run) p_reg = sub.add_parser("regression", help="Run the golden-dataset regression eval") p_reg.set_defaults(func=cmd_regression) p_status = sub.add_parser("status", help="Show golden / flagged counts") p_status.set_defaults(func=cmd_status) args = parser.parse_args() return args.func(args) if __name__ == "__main__": raise SystemExit(main())