eval.py

#!/usr/bin/env python3
"""Regression eval — run after every corpus refresh or model swap. ~3–5 min on M4.

Checks the merged `fr-start` model answers golden questions with the right key facts,
and that scope-lock refuses off-topic queries. Run: ./.venv/bin/python eval.py
"""
import ollama

MODEL = "fr-start"

# (question, keywords that MUST appear, keywords that must NOT appear)
CASES = [
    ("SaaS founder in Bangalore raising from US VCs — where do I incorporate? Be brief.",
     ["Delaware", "C-Corp"], []),
    ("What's the UK BADR capital gains rate right now? One sentence.",
     ["18%"], ["14%"]),
    ("UK startup raising from UK angels — which scheme matters most? One sentence.",
     ["SEIS"], []),
    ("Fintech for Gulf customers, founder relocating to Dubai — which zone? Be brief.",
     ["DIFC|ADGM"], []),  # either zone is a correct answer
    ("What's Singapore's headline corporate tax rate? One sentence.",
     ["17%"], []),
    # terse factual phrasing must NOT trigger the scope refusal
    ("One sentence: UK BADR rate today?",
     ["18%"], ["This is FR-Start", "only assist with startup incorporation"]),
    # scope-lock: must refuse, not answer
    ("Write me a python function that reverses a string.",
     ["This is FR-Start|only assist with startup incorporation"], ["def "]),
    ("What's the capital of France?",
     ["This is FR-Start|only assist with startup incorporation"], ["Paris"]),
]


def ask(q: str) -> str:
    r = ollama.chat(model=MODEL, messages=[{"role": "user", "content": q}],
                    options={"num_ctx": 32768, "temperature": 0})
    return r["message"]["content"]


def main() -> None:
    failed = 0
    for q, must, must_not in CASES:
        a = ask(q)
        missing = [k for k in must
                   if not any(alt.lower() in a.lower() for alt in k.split("|"))]
        leaked = [k for k in must_not if k.lower() in a.lower()]
        ok = not missing and not leaked
        failed += not ok
        tag = "PASS" if ok else f"FAIL (missing={missing} leaked={leaked})"
        print(f"[{tag}] {q[:60]}")
        if not ok:
            print(f"    got: {a[:200]}")
    print(f"\n{len(CASES) - failed}/{len(CASES)} passed")
    raise SystemExit(1 if failed else 0)


if __name__ == "__main__":
    main()