Spaces:
Sleeping
Sleeping
File size: 3,522 Bytes
3be03dd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | """
eval.runner — shared test infrastructure for the evaluation suite.
Provides:
ROOT — absolute path to the repository root (sys.path entry)
ANSI colours — GREEN, RED, YELLOW, CYAN, BOLD, DIM, RESET, and icons
EvalResult — records per-test outcomes
_results — module-level singleton shared by all suites
run_test() — execute one test, print result, record it
section() — print a bold section header
"""
import os
import sys
import time
import traceback
from typing import Callable
# ── Make repo root importable when running from any directory ───────────────
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if ROOT not in sys.path:
sys.path.insert(0, ROOT)
# ── ANSI colours ─────────────────────────────────────────────────────────────
GREEN = "\033[92m"
RED = "\033[91m"
YELLOW = "\033[93m"
CYAN = "\033[96m"
BOLD = "\033[1m"
DIM = "\033[2m"
RESET = "\033[0m"
PASS_ICON = f"{GREEN}✓{RESET}"
FAIL_ICON = f"{RED}✗{RESET}"
SKIP_ICON = f"{YELLOW}○{RESET}"
RUN_ICON = f"{CYAN}→{RESET}"
# ── EvalResult ────────────────────────────────────────────────────────────────
class EvalResult:
def __init__(self):
self.results: list[dict] = []
def record(self, suite: str, name: str, passed: bool,
elapsed: float, detail: str = ""):
self.results.append(dict(suite=suite, name=name, passed=passed,
elapsed=elapsed, detail=detail))
def passed(self): return [r for r in self.results if r["passed"]]
def failed(self): return [r for r in self.results if not r["passed"]]
def count(self): return len(self.results)
# Module-level singleton — imported by all suite files so results accumulate.
_results = EvalResult()
# ── Test primitives ──────────────────────────────────────────────────────────
def run_test(suite: str, name: str, fn: Callable,
skip_if: bool = False, skip_reason: str = "") -> bool:
"""Run one test, print result, record it in _results."""
label = f"{DIM}{suite:<12}{RESET} {name}"
if skip_if:
print(f" {SKIP_ICON} {label:<55} {YELLOW}SKIP{RESET} {DIM}{skip_reason}{RESET}")
return True
print(f" {RUN_ICON} {label:<55}", end="", flush=True)
t0 = time.time()
try:
detail = fn() or ""
elapsed = time.time() - t0
_results.record(suite, name, True, elapsed, str(detail)[:120])
print(f"\r {PASS_ICON} {label:<55} {GREEN}{elapsed:5.2f}s{RESET} {DIM}{detail}{RESET}")
return True
except Exception as exc:
elapsed = time.time() - t0
short = f"{type(exc).__name__}: {str(exc)[:80]}"
_results.record(suite, name, False, elapsed, short)
print(f"\r {FAIL_ICON} {label:<55} {RED}{elapsed:5.2f}s{RESET} {RED}{short}{RESET}")
if os.getenv("EVAL_VERBOSE"):
traceback.print_exc()
return False
def section(title: str):
print(f"\n{BOLD}{CYAN}{'─'*70}{RESET}")
print(f"{BOLD}{CYAN} {title}{RESET}")
print(f"{BOLD}{CYAN}{'─'*70}{RESET}")
|