test_eval_benchmarks.py

"""Offline quality gate for the fixed evaluation benchmark; does not require app.py or a model."""
import importlib.util
from pathlib import Path

HERE = Path(__file__).resolve().parent
spec = importlib.util.spec_from_file_location("eval_eim", HERE / "eval_eim.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
problems = module.PROBLEMS
assert len(problems) >= 25, f"benchmark too small: {len(problems)}"
ids = [p["id"] for p in problems]
assert len(ids) == len(set(ids)), "duplicate benchmark IDs"
count = 0
for problem in problems:
    assert len(problem["tests"]) >= 5, f"{problem['id']} has too few tests"
    ns = {}
    exec(problem["reference"], ns)
    for test in problem["tests"]:
        exec(test, ns)
        count += 1
print(f"PASS: {len(problems)} unique problems; all {count} reference assertions pass.")