eval/runner.py

"""
eval.runner — shared test infrastructure for the evaluation suite.

Provides:
  ROOT          — absolute path to the repository root (sys.path entry)
  ANSI colours  — GREEN, RED, YELLOW, CYAN, BOLD, DIM, RESET, and icons
  EvalResult    — records per-test outcomes
  _results      — module-level singleton shared by all suites
  run_test()    — execute one test, print result, record it
  section()     — print a bold section header
"""
import os
import sys
import time
import traceback
from typing import Callable

# ── Make repo root importable when running from any directory ───────────────
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if ROOT not in sys.path:
    sys.path.insert(0, ROOT)

# ── ANSI colours ─────────────────────────────────────────────────────────────
GREEN  = "\033[92m"
RED    = "\033[91m"
YELLOW = "\033[93m"
CYAN   = "\033[96m"
BOLD   = "\033[1m"
DIM    = "\033[2m"
RESET  = "\033[0m"

PASS_ICON = f"{GREEN}✓{RESET}"
FAIL_ICON = f"{RED}✗{RESET}"
SKIP_ICON = f"{YELLOW}○{RESET}"
RUN_ICON  = f"{CYAN}→{RESET}"


# ── EvalResult ────────────────────────────────────────────────────────────────

class EvalResult:
    def __init__(self):
        self.results: list[dict] = []

    def record(self, suite: str, name: str, passed: bool,
               elapsed: float, detail: str = ""):
        self.results.append(dict(suite=suite, name=name, passed=passed,
                                  elapsed=elapsed, detail=detail))

    def passed(self):  return [r for r in self.results if r["passed"]]
    def failed(self):  return [r for r in self.results if not r["passed"]]
    def count(self):   return len(self.results)


# Module-level singleton — imported by all suite files so results accumulate.
_results = EvalResult()


# ── Test primitives ──────────────────────────────────────────────────────────

def run_test(suite: str, name: str, fn: Callable,
             skip_if: bool = False, skip_reason: str = "") -> bool:
    """Run one test, print result, record it in _results."""
    label = f"{DIM}{suite:<12}{RESET} {name}"
    if skip_if:
        print(f"  {SKIP_ICON}  {label:<55} {YELLOW}SKIP{RESET}  {DIM}{skip_reason}{RESET}")
        return True

    print(f"  {RUN_ICON}  {label:<55}", end="", flush=True)
    t0 = time.time()
    try:
        detail  = fn() or ""
        elapsed = time.time() - t0
        _results.record(suite, name, True, elapsed, str(detail)[:120])
        print(f"\r  {PASS_ICON}  {label:<55} {GREEN}{elapsed:5.2f}s{RESET}  {DIM}{detail}{RESET}")
        return True
    except Exception as exc:
        elapsed = time.time() - t0
        short   = f"{type(exc).__name__}: {str(exc)[:80]}"
        _results.record(suite, name, False, elapsed, short)
        print(f"\r  {FAIL_ICON}  {label:<55} {RED}{elapsed:5.2f}s{RESET}  {RED}{short}{RESET}")
        if os.getenv("EVAL_VERBOSE"):
            traceback.print_exc()
        return False


def section(title: str):
    print(f"\n{BOLD}{CYAN}{'─'*70}{RESET}")
    print(f"{BOLD}{CYAN}  {title}{RESET}")
    print(f"{BOLD}{CYAN}{'─'*70}{RESET}")