Spaces:
Sleeping
Sleeping
File size: 5,451 Bytes
d402333 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | """
Regression tests — one test per failure in the feedback.
Runs with the RULES backend (no model download) so it's fast and CI-able.
The LLM backends only improve on these results.
"""
import sys
import logging
logging.basicConfig(level=logging.WARNING)
from nlu import NLU
from orchestrator import Orchestrator, fmt_amount
PASS, FAIL = "\033[92mPASS\033[0m", "\033[91mFAIL\033[0m"
results = []
def check(name: str, cond: bool, detail: str = ""):
results.append((name, cond))
print(f" [{PASS if cond else FAIL}] {name}" + (f" — {detail}" if detail else ""))
def new_orch():
return Orchestrator(nlu=NLU(prefer="rules"))
def turn(orch, session, text):
resp, session, esc = orch.respond(text, text, session)
return resp, session, esc
print("\n══ P1: 'send money to abu now' must NOT re-ask for recipient ══")
orch = new_orch()
s = orch.new_session()
resp, s, _ = turn(orch, s, "I want to send money to abu now")
check("recipient 'abu' prefilled (no 'who do you want to send to')",
"who" not in resp.lower() and "abu" not in resp.lower().replace("abu", "") or
("recipient" not in resp.lower() and "who would you like to send" not in resp.lower()),
f"agent said: {resp[:90]}")
check("asks for the AMOUNT instead (the actually-missing slot)",
"how much" in resp.lower(), f"agent said: {resp[:90]}")
print("\n══ P0: compound 'check my balance and also send 350000 to amina' ══")
orch = new_orch()
s = orch.new_session()
resp, s, _ = turn(orch, s, "check my balance and also send 350000 to amina")
check("acknowledges BOTH tasks", "one at a time" in resp.lower() or
("balance" in resp.lower() and ("transfer" in resp.lower() or "amina" in resp.lower())),
f"agent said: {resp[:120]}")
# give account number → balance executes, transfer flow continues
resp, s, _ = turn(orch, s, "1234567890")
check("balance is delivered", "balance is" in resp.lower(), f"{resp[:90]}")
check("transfer to amina NOT dropped (asks to confirm or continues it)",
"amina" in resp.lower() or "send" in resp.lower() or "confirm" in resp.lower()
or "yes" in resp.lower(),
f"{resp[:140]}")
print("\n══ P0: 'where is your closest branch so I can get my ATM card' ══")
orch = new_orch()
s = orch.new_session()
resp, s, _ = turn(orch, s, "where is your closest branch so I can get my ATM card")
check("NOT classified as block_card (no 'block' confirmation)",
"block" not in resp.lower(), f"{resp[:120]}")
check("gives branch info", "branch" in resp.lower() or "open" in resp.lower(),
f"{resp[:120]}")
print("\n══ P1: return reason 'too small' must not dead-end ══")
orch = new_orch()
s = orch.new_session()
resp, s, _ = turn(orch, s, "I want to return my order")
resp, s, _ = turn(orch, s, "ORD-4521")
resp, s, _ = turn(orch, s, "too small")
check("'too small' accepted as return reason",
"too small" in resp.lower() or "return" in resp.lower(),
f"{resp[:120]}")
check("no 'did not understand' dead-end",
"didn't understand" not in resp.lower() and "did not understand" not in resp.lower(),
f"{resp[:90]}")
print("\n══ P1: unknown input → clarify once, then human WITH context ══")
orch = new_orch()
s = orch.new_session()
resp, s, esc = turn(orch, s, "florble the wumbus")
check("first unknown → clarifying question, not menu dump",
"are you asking" in resp.lower() or "balance" in resp.lower(), f"{resp[:110]}")
check("not escalated yet", not esc)
resp, s, esc = turn(orch, s, "zorp zorp quux")
check("second unknown → human handoff", esc, f"{resp[:110]}")
check("handoff mentions context is preserved",
"won't have to repeat" in resp.lower() or "conversation" in resp.lower(),
f"{resp[:140]}")
print("\n══ P2: balance guard — 350,000 against smaller balance ══")
orch = new_orch()
s = orch.new_session()
resp, s, _ = turn(orch, s, "check my balance")
resp, s, _ = turn(orch, s, "1111111") # deterministic mock balance
bal = s.balance
too_much = bal + 100_000
resp, s, _ = turn(orch, s, f"send {too_much} to musa")
check("over-balance transfer is refused, not confirmed",
"can't process" in resp.lower() or "balance is" in resp.lower(),
f"balance={fmt_amount(bal)}, asked={fmt_amount(too_much)} → {resp[:130]}")
print("\n══ P2: consistent ₦ formatting ══")
check("fmt_amount('350000') == '₦350,000'", fmt_amount("350000") == "₦350,000")
check("fmt_amount('134200') == '₦134,200'", fmt_amount("134200") == "₦134,200")
check("fmt_amount(245000) == '₦245,000'", fmt_amount(245000) == "₦245,000")
print("\n══ Destructive confirmation gate ══")
orch = new_orch()
s = orch.new_session()
resp, s, _ = turn(orch, s, "send 5000 to fatima")
resp, s, _ = turn(orch, s, "1234567890") # account slot
check("transfer requires explicit yes/no before executing",
"yes" in resp.lower() and "confirm" in resp.lower(), f"{resp[:130]}")
resp, s, _ = turn(orch, s, "no")
check("'no' cancels cleanly", "cancelled" in resp.lower(), f"{resp[:90]}")
# ── Summary ────────────────────────────────────────────────────────────────────
passed = sum(1 for _, ok in results if ok)
total = len(results)
print(f"\n{'='*60}\n {passed}/{total} checks passed\n{'='*60}")
sys.exit(0 if passed == total else 1)
|