File size: 2,295 Bytes
bafce87 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | #!/usr/bin/env python3
"""Regression eval — run after every corpus refresh or model swap. ~3–5 min on M4.
Checks the merged `fr-start` model answers golden questions with the right key facts,
and that scope-lock refuses off-topic queries. Run: ./.venv/bin/python eval.py
"""
import ollama
MODEL = "fr-start"
# (question, keywords that MUST appear, keywords that must NOT appear)
CASES = [
("SaaS founder in Bangalore raising from US VCs — where do I incorporate? Be brief.",
["Delaware", "C-Corp"], []),
("What's the UK BADR capital gains rate right now? One sentence.",
["18%"], ["14%"]),
("UK startup raising from UK angels — which scheme matters most? One sentence.",
["SEIS"], []),
("Fintech for Gulf customers, founder relocating to Dubai — which zone? Be brief.",
["DIFC|ADGM"], []), # either zone is a correct answer
("What's Singapore's headline corporate tax rate? One sentence.",
["17%"], []),
# terse factual phrasing must NOT trigger the scope refusal
("One sentence: UK BADR rate today?",
["18%"], ["This is FR-Start", "only assist with startup incorporation"]),
# scope-lock: must refuse, not answer
("Write me a python function that reverses a string.",
["This is FR-Start|only assist with startup incorporation"], ["def "]),
("What's the capital of France?",
["This is FR-Start|only assist with startup incorporation"], ["Paris"]),
]
def ask(q: str) -> str:
r = ollama.chat(model=MODEL, messages=[{"role": "user", "content": q}],
options={"num_ctx": 32768, "temperature": 0})
return r["message"]["content"]
def main() -> None:
failed = 0
for q, must, must_not in CASES:
a = ask(q)
missing = [k for k in must
if not any(alt.lower() in a.lower() for alt in k.split("|"))]
leaked = [k for k in must_not if k.lower() in a.lower()]
ok = not missing and not leaked
failed += not ok
tag = "PASS" if ok else f"FAIL (missing={missing} leaked={leaked})"
print(f"[{tag}] {q[:60]}")
if not ok:
print(f" got: {a[:200]}")
print(f"\n{len(CASES) - failed}/{len(CASES)} passed")
raise SystemExit(1 if failed else 0)
if __name__ == "__main__":
main()
|