Build 100-case scam eval spine

Co-authored-by: Codex <codex@openai.com>

CODEX_BUILD_LOG.md

@@ -21,3 +21,6 @@ Repository evidence:
 
 Going forward, meaningful build commits should include a Codex co-author trailer in the Git commit message.
 
+## Eval Spine
+
+Codex helped expand the initial seed eval into a 100-case labeled dataset and upgraded the runner to report risk accuracy, scam-type accuracy, tactic recall, dangerous misses, safe-message false alarms, unsafe action violations, and category-level performance.

FIELD_NOTES.md

@@ -15,3 +15,4 @@ Current risk: a generic scam detector will not stand out. The product must show
 
 Current differentiator: Scam DNA, a visual breakdown of scam structure rather than a plain label.
 
+Built the first serious evaluation spine: 100 synthetic/sanitized scam, suspicious, needs-check, and safe messages. The eval explicitly tests false positives on legitimate-looking messages because user trust is central to the product.

SUBMISSION.md

@@ -19,7 +19,7 @@
 ## Technical Evidence
 
 - [ ] Model name and parameter count documented
-- [ ] Eval set included
+- [x] Eval set included
 - [ ] Bakeoff results documented
 - [ ] Local inference path documented
 - [ ] No cloud API path documented

eval/README.md

@@ -0,0 +1,23 @@
+# Jawbreaker Eval Set
+
+`scam_eval.jsonl` is the project compass for model selection.
+
+The first version contains 100 synthetic/sanitized examples across:
+
+- dangerous scams
+- suspicious messages
+- legitimate messages that still need verification
+- safe benign messages
+
+Primary metrics:
+
+- valid structured output
+- exact risk-level match
+- dangerous scams mislabeled as safe
+- safe messages mislabeled as dangerous or suspicious
+- action safety
+- tactic recall
+- latency
+
+The eval intentionally includes legitimate alerts and ordinary messages. A scam detector that calls everything dangerous is not useful for the person Jawbreaker is built to protect.
+

eval/run_eval.py

@@ -1,24 +1,150 @@
 import json
 import sys
+from collections import Counter, defaultdict
 from pathlib import Path
+from time import perf_counter
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
-from jawbreaker.schema import ScamAnalysis
+from jawbreaker.schema import RISK_LEVELS, ScamAnalysis
+
+
+UNSAFE_ACTION_PHRASES = [
+    "click the link",
+    "open the link",
+    "use the link",
+    "call the number in the message",
+    "call the number shown",
+    "reply with your code",
+    "send the code",
+    "send money",
+    "buy gift cards",
+    "share your password",
+    "share your pin",
+]
+
+
+def load_rows(path: Path) -> list[dict]:
+    rows = []
+    ids = set()
+    errors = []
+
+    for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError as exc:
+            errors.append(f"line {line_number}: invalid JSON: {exc}")
+            continue
+
+        missing = {"id", "category", "input", "expected_risk_level", "expected_scam_type", "expected_tactics"} - set(row)
+        if missing:
+            errors.append(f"line {line_number}: missing fields: {sorted(missing)}")
+
+        if row.get("id") in ids:
+            errors.append(f"line {line_number}: duplicate id: {row.get('id')}")
+        ids.add(row.get("id"))
+
+        if row.get("expected_risk_level") not in RISK_LEVELS:
+            errors.append(f"line {line_number}: invalid expected_risk_level: {row.get('expected_risk_level')}")
+
+        if not isinstance(row.get("expected_tactics"), list):
+            errors.append(f"line {line_number}: expected_tactics must be a list")
+
+        rows.append(row)
+
+    if errors:
+        raise SystemExit("Eval dataset validation failed:\n" + "\n".join(errors))
+    return rows
+
+
+def has_unsafe_action(action: str) -> bool:
+    text = action.lower()
+    for phrase in UNSAFE_ACTION_PHRASES:
+        if phrase not in text:
+            continue
+        if f"do not {phrase}" in text or f"never {phrase}" in text or f"don't {phrase}" in text:
+            continue
+        return True
+    return False
+
+
+def tactic_recall(expected: list[str], actual: list[str]) -> float:
+    if not expected:
+        return 1.0
+    expected_set = set(expected)
+    actual_set = set(actual)
+    return len(expected_set & actual_set) / len(expected_set)
 
 
 def main() -> None:
     path = Path(__file__).with_name("scam_eval.jsonl")
-    rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
-    correct = 0
+    rows = load_rows(path)
+    started = perf_counter()
+
+    risk_correct = 0
+    scam_type_correct = 0
+    dangerous_as_safe = []
+    safe_as_dangerous = []
+    unsafe_actions = []
+    recalls = []
+    risk_confusion: Counter[tuple[str, str]] = Counter()
+    category_counts: Counter[str] = Counter()
+    category_correct: Counter[str] = Counter()
+    failures_by_category: dict[str, list[str]] = defaultdict(list)
 
     for row in rows:
         analysis = ScamAnalysis.from_heuristics(row["input"])
-        ok = analysis.risk_level == row["expected_risk_level"]
-        correct += int(ok)
-        print(f"{row['id']}: expected={row['expected_risk_level']} actual={analysis.risk_level} ok={ok}")
+        expected_risk = row["expected_risk_level"]
+        category = row["category"]
+        risk_ok = analysis.risk_level == expected_risk
+        type_ok = analysis.scam_type == row["expected_scam_type"]
+        recall = tactic_recall(row["expected_tactics"], analysis.tactics)
+
+        risk_correct += int(risk_ok)
+        scam_type_correct += int(type_ok)
+        recalls.append(recall)
+        risk_confusion[(expected_risk, analysis.risk_level)] += 1
+        category_counts[category] += 1
+        category_correct[category] += int(risk_ok)
+
+        if expected_risk == "dangerous" and analysis.risk_level == "safe":
+            dangerous_as_safe.append(row["id"])
+        if expected_risk == "safe" and analysis.risk_level in {"dangerous", "suspicious"}:
+            safe_as_dangerous.append(row["id"])
+        if has_unsafe_action(analysis.safest_action):
+            unsafe_actions.append(row["id"])
+        if not risk_ok:
+            failures_by_category[category].append(f"{row['id']} expected={expected_risk} actual={analysis.risk_level}")
+
+    elapsed = perf_counter() - started
+    total = len(rows)
+
+    print(f"cases={total}")
+    print(f"risk_level_accuracy={risk_correct}/{total} ({risk_correct / total:.1%})")
+    print(f"scam_type_accuracy={scam_type_correct}/{total} ({scam_type_correct / total:.1%})")
+    print(f"mean_tactic_recall={sum(recalls) / len(recalls):.1%}")
+    print(f"dangerous_as_safe={len(dangerous_as_safe)} {dangerous_as_safe}")
+    print(f"safe_as_dangerous_or_suspicious={len(safe_as_dangerous)} {safe_as_dangerous}")
+    print(f"unsafe_action_violations={len(unsafe_actions)} {unsafe_actions}")
+    print(f"elapsed_seconds={elapsed:.3f}")
+
+    print("\nrisk_confusion expected->actual:")
+    for (expected, actual), count in sorted(risk_confusion.items()):
+        print(f"  {expected:12s} -> {actual:12s} {count}")
+
+    print("\ncategory_risk_accuracy:")
+    for category, count in sorted(category_counts.items()):
+        correct = category_correct[category]
+        print(f"  {category:24s} {correct:2d}/{count:2d} ({correct / count:.1%})")
 
-    print(f"risk_level_accuracy={correct}/{len(rows)}")
+    if failures_by_category:
+        print("\nfirst_failures_by_category:")
+        for category, failures in sorted(failures_by_category.items()):
+            print(f"  {category}:")
+            for failure in failures[:5]:
+                print(f"    {failure}")
 
 
 if __name__ == "__main__":

eval/scam_eval.jsonl

@@ -1,7 +1,100 @@
-{"id":"package_phish_001","input":"USPS: Your package is held due to an unpaid fee. Verify now: http://usps-track-secure.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["urgency","suspicious link","fake authority"]}
-{"id":"family_impersonation_001","input":"Hi Grandma, I lost my phone. This is my new number. Can you send $800 for rent today? Please don't tell Mom.","expected_risk_level":"dangerous","expected_scam_type":"family_impersonation","expected_tactics":["impersonation","urgency","secrecy","payment pressure"]}
-{"id":"bank_safe_001","input":"Chase fraud alert: Did you attempt a $249.00 purchase at TARGET? Reply YES or NO.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_alert","expected_tactics":["verification needed"]}
-{"id":"gift_card_001","input":"This is your manager. I need you to buy three gift cards immediately and send me the codes. I am in a meeting.","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["fake authority","urgency","payment pressure"]}
-{"id":"safe_friend_001","input":"Running ten minutes late. See you at the restaurant soon.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
-{"id":"password_reset_001","input":"Your email password expires today. Login now to keep access: https://mail-security-reset.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["urgency","suspicious link","credential request"]}
-
+{"id":"package_phish_001","category":"package_phishing","input":"USPS: Your package is held due to an unpaid fee. Verify now: http://usps-track-secure.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","urgency","suspicious link","credential request"]}
+{"id":"package_phish_002","category":"package_phishing","input":"FedEx notice: delivery failed. Confirm your address and card details before 6 PM: https://fedex-redelivery-check.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","urgency","suspicious link","payment request"]}
+{"id":"package_phish_003","category":"package_phishing","input":"Your parcel is waiting at customs. Pay the $2.14 clearance fee here: http://customs-parcel-pay.example","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["fake authority","small fee","suspicious link"]}
+{"id":"package_phish_004","category":"package_phishing","input":"UPS: We cannot deliver your item until you update your login information: https://ups-account-update.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","suspicious link","credential request"]}
+{"id":"package_phish_005","category":"package_phishing","input":"Postal Service: final attempt today. Your package will be returned unless you verify your details: http://post-office-secure.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","urgency","suspicious link","credential request"]}
+{"id":"package_phish_006","category":"package_phishing","input":"DHL Express: tax unpaid. Pay now or your package will be destroyed: https://dhl-tax-release.example","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["fake authority","urgency","threat","suspicious link"]}
+{"id":"package_phish_007","category":"package_phishing","input":"Your Amazon order is delayed. Confirm payment at https://amazon-billing-help.example to avoid cancellation.","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","urgency","suspicious link","credential request"]}
+{"id":"package_phish_008","category":"package_phishing","input":"Package Center: incomplete address. Enter your full name, birth date, and card number here: http://delivery-address-fix.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["suspicious link","credential request","payment request"]}
+{"id":"bank_phish_001","category":"bank_phishing","input":"Bank alert: your account is locked. Login immediately to restore access: https://secure-bank-restore.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","urgency","suspicious link","credential request"]}
+{"id":"bank_phish_002","category":"bank_phishing","input":"Fraud department: unusual transfer detected. Verify your password and one-time code now: http://fraud-review-bank.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","urgency","suspicious link","credential request"]}
+{"id":"bank_phish_003","category":"bank_phishing","input":"Your debit card will be suspended in 30 minutes. Confirm your PIN here: https://card-security-check.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","urgency","suspicious link","credential request"]}
+{"id":"bank_phish_004","category":"bank_phishing","input":"Security notice: we blocked a $940 wire. Reply with your full card number to cancel it.","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","urgency","credential request"]}
+{"id":"bank_phish_005","category":"bank_phishing","input":"Your mobile banking profile expires today. Re-enroll at http://mobile-bank-profile.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","urgency","suspicious link","credential request"]}
+{"id":"bank_phish_006","category":"bank_phishing","input":"Credit union support: your account has been flagged. Send your username so we can verify ownership.","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","credential request"]}
+{"id":"bank_phish_007","category":"bank_phishing","input":"Payment failed. Update your wallet seed phrase to keep your funds safe: https://wallet-support-reset.example","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["fake authority","suspicious link","credential request"]}
+{"id":"bank_phish_008","category":"bank_phishing","input":"Zelle support: a $1,200 transfer is pending. To cancel, send $1 to yourself using the instructions at https://zelle-cancel-flow.example","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["fake authority","payment request","suspicious link","urgency"]}
+{"id":"family_impersonation_001","category":"family_impersonation","input":"Hi Grandma, I lost my phone. This is my new number. Can you send $800 for rent today? Please don't tell Mom.","expected_risk_level":"dangerous","expected_scam_type":"family_impersonation","expected_tactics":["impersonation","urgency","secrecy","payment pressure"]}
+{"id":"family_impersonation_002","category":"family_impersonation","input":"Mom it's me. I got in a wreck and need bail money. Please wire $1,500 now and don't call anyone.","expected_risk_level":"dangerous","expected_scam_type":"family_impersonation","expected_tactics":["impersonation","urgency","secrecy","payment pressure","emotional manipulation"]}
+{"id":"family_impersonation_003","category":"family_impersonation","input":"Grandpa, my phone broke. Text me on this number only. I need help paying a bill before midnight.","expected_risk_level":"dangerous","expected_scam_type":"family_impersonation","expected_tactics":["impersonation","urgency","new channel"]}
+{"id":"family_impersonation_004","category":"family_impersonation","input":"Auntie please don't tell Dad. I need gift cards for an emergency and I'll pay you back tomorrow.","expected_risk_level":"dangerous","expected_scam_type":"family_impersonation","expected_tactics":["impersonation","secrecy","payment pressure","emotional manipulation"]}
+{"id":"family_impersonation_005","category":"family_impersonation","input":"This is your son using a friend's phone. I am stranded. Can you send crypto to this wallet right away?","expected_risk_level":"dangerous","expected_scam_type":"family_impersonation","expected_tactics":["impersonation","urgency","payment pressure","new channel"]}
+{"id":"family_impersonation_006","category":"family_impersonation","input":"Hi, this is your niece. I changed numbers. Can you send me the code that just came to your phone?","expected_risk_level":"dangerous","expected_scam_type":"account_takeover","expected_tactics":["impersonation","new channel","credential request"]}
+{"id":"family_impersonation_007","category":"family_impersonation","input":"Grandma I am in trouble. Please send money through Zelle today. I cannot talk on the phone.","expected_risk_level":"dangerous","expected_scam_type":"family_impersonation","expected_tactics":["impersonation","urgency","payment pressure","refuses verification"]}
+{"id":"gift_card_001","category":"gift_card","input":"This is your manager. I need you to buy three gift cards immediately and send me the codes. I am in a meeting.","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["fake authority","urgency","payment pressure","gift cards"]}
+{"id":"gift_card_002","category":"gift_card","input":"Can you do me a favor? Pick up $500 in Apple cards and scratch the backs. I need the photos ASAP.","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["payment pressure","gift cards","urgency"]}
+{"id":"gift_card_003","category":"gift_card","input":"Past-due tax notice: settle today with prepaid cards to avoid legal action.","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["fake authority","threat","urgency","gift cards"]}
+{"id":"gift_card_004","category":"gift_card","input":"Your computer repair invoice must be paid with Google Play cards before service can continue.","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["payment pressure","gift cards","fake invoice"]}
+{"id":"gift_card_005","category":"gift_card","input":"School fundraiser urgent: send gift card codes to this temporary number so your child keeps their spot.","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["fake authority","urgency","gift cards","emotional manipulation"]}
+{"id":"gift_card_006","category":"gift_card","input":"Payroll issue: buy $300 in Steam cards and email the codes to verify your direct deposit.","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["fake authority","gift cards","credential request","payment pressure"]}
+{"id":"gift_card_007","category":"gift_card","input":"I am the pastor. A family needs help now. Please send the gift card numbers to this phone.","expected_risk_level":"dangerous","expected_scam_type":"payment_request","expected_tactics":["fake authority","emotional manipulation","gift cards","new channel"]}
+{"id":"tech_support_001","category":"tech_support","input":"Microsoft Security: your device is infected. Call the support number on this message within 10 minutes.","expected_risk_level":"dangerous","expected_scam_type":"tech_support","expected_tactics":["fake authority","urgency","threat","callback request"]}
+{"id":"tech_support_002","category":"tech_support","input":"Your antivirus subscription renewed for $399. To cancel, download the refund tool: http://refund-support-tool.example","expected_risk_level":"dangerous","expected_scam_type":"tech_support","expected_tactics":["fake invoice","suspicious link","refund bait"]}
+{"id":"tech_support_003","category":"tech_support","input":"Apple support: we found hackers on your iCloud. Reply with the 6-digit code to remove them.","expected_risk_level":"dangerous","expected_scam_type":"account_takeover","expected_tactics":["fake authority","credential request","fear"]}
+{"id":"tech_support_004","category":"tech_support","input":"Browser warning: your files are being deleted. Call certified support now at the number shown here.","expected_risk_level":"dangerous","expected_scam_type":"tech_support","expected_tactics":["fear","urgency","callback request"]}
+{"id":"tech_support_005","category":"tech_support","input":"Geek Help Desk invoice: $289 charge approved. If this was not you, call this number for a refund.","expected_risk_level":"dangerous","expected_scam_type":"refund_scam","expected_tactics":["fake invoice","callback request","refund bait"]}
+{"id":"prize_001","category":"prize_lottery","input":"Congratulations! You won an iPhone. Pay a $9.99 shipping fee here: https://winner-shipping.example","expected_risk_level":"dangerous","expected_scam_type":"prize_scam","expected_tactics":["too good to be true","payment request","suspicious link"]}
+{"id":"prize_002","category":"prize_lottery","input":"Lottery office: your unclaimed prize is ready. Send your bank login so we can deposit winnings.","expected_risk_level":"dangerous","expected_scam_type":"prize_scam","expected_tactics":["too good to be true","fake authority","credential request"]}
+{"id":"prize_003","category":"prize_lottery","input":"You have been selected for a government grant. Processing fee required by midnight: http://grant-release.example","expected_risk_level":"dangerous","expected_scam_type":"advance_fee","expected_tactics":["fake authority","payment request","urgency","suspicious link"]}
+{"id":"prize_004","category":"prize_lottery","input":"Claim your free vacation now. We only need your card number to hold the reservation.","expected_risk_level":"dangerous","expected_scam_type":"prize_scam","expected_tactics":["too good to be true","payment request","urgency"]}
+{"id":"prize_005","category":"prize_lottery","input":"Final notice: your reward points expire today. Login at https://points-claim-center.example to cash out.","expected_risk_level":"dangerous","expected_scam_type":"credential_theft","expected_tactics":["urgency","suspicious link","credential request"]}
+{"id":"romance_001","category":"romance","input":"I feel so close to you. My wallet was stolen while traveling. Can you send $300 so I can get home?","expected_risk_level":"dangerous","expected_scam_type":"romance_scam","expected_tactics":["emotional manipulation","payment pressure","urgency"]}
+{"id":"romance_002","category":"romance","input":"I want to visit you, but the travel agent needs payment today. Please send it by wire transfer.","expected_risk_level":"dangerous","expected_scam_type":"romance_scam","expected_tactics":["emotional manipulation","payment pressure","urgency"]}
+{"id":"romance_003","category":"romance","input":"Do not tell your family about us yet. I need help with a hospital bill before we can meet.","expected_risk_level":"dangerous","expected_scam_type":"romance_scam","expected_tactics":["secrecy","emotional manipulation","payment pressure"]}
+{"id":"job_scam_001","category":"job_scam","input":"Remote job approved. Deposit this check and send part of it to the equipment vendor today.","expected_risk_level":"dangerous","expected_scam_type":"fake_check","expected_tactics":["fake job","payment pressure","overpayment"]}
+{"id":"job_scam_002","category":"job_scam","input":"You are hired. Pay a refundable $75 onboarding fee at https://newhire-start.example to unlock training.","expected_risk_level":"dangerous","expected_scam_type":"advance_fee","expected_tactics":["fake job","payment request","suspicious link"]}
+{"id":"suspicious_link_001","category":"suspicious","input":"Your account needs attention. Visit http://account-message-center.example when you have time.","expected_risk_level":"suspicious","expected_scam_type":"unknown_link","expected_tactics":["suspicious link","vague request"]}
+{"id":"suspicious_link_002","category":"suspicious","input":"We could not process your request. Open https://customer-resolution.example for details.","expected_risk_level":"suspicious","expected_scam_type":"unknown_link","expected_tactics":["suspicious link","vague request"]}
+{"id":"suspicious_link_003","category":"suspicious","input":"Your profile is incomplete. Confirm your information here: http://profile-update-center.example","expected_risk_level":"suspicious","expected_scam_type":"possible_credential_theft","expected_tactics":["suspicious link","credential request"]}
+{"id":"suspicious_link_004","category":"suspicious","input":"Please review the secure document I shared: https://document-viewer-secure.example","expected_risk_level":"suspicious","expected_scam_type":"unknown_link","expected_tactics":["suspicious link","vague request"]}
+{"id":"suspicious_link_005","category":"suspicious","input":"Your voicemail is available at http://voice-message-download.example","expected_risk_level":"suspicious","expected_scam_type":"unknown_link","expected_tactics":["suspicious link","attachment bait"]}
+{"id":"suspicious_urgency_001","category":"suspicious","input":"Please call me back immediately about your account. This cannot wait.","expected_risk_level":"suspicious","expected_scam_type":"unknown_callback","expected_tactics":["urgency","vague request"]}
+{"id":"suspicious_urgency_002","category":"suspicious","input":"Your subscription has a problem. We need confirmation today or it may be closed.","expected_risk_level":"suspicious","expected_scam_type":"unknown_billing","expected_tactics":["urgency","vague request"]}
+{"id":"suspicious_urgency_003","category":"suspicious","input":"Final reminder: your paperwork is missing. Send your details by end of day.","expected_risk_level":"suspicious","expected_scam_type":"possible_credential_theft","expected_tactics":["urgency","credential request","vague request"]}
+{"id":"suspicious_urgency_004","category":"suspicious","input":"Action required: confirm ownership of your account within 24 hours.","expected_risk_level":"suspicious","expected_scam_type":"possible_credential_theft","expected_tactics":["urgency","credential request"]}
+{"id":"suspicious_urgency_005","category":"suspicious","input":"We noticed an issue with your delivery. Respond quickly so we can resolve it.","expected_risk_level":"suspicious","expected_scam_type":"possible_package_scam","expected_tactics":["urgency","vague request"]}
+{"id":"suspicious_sender_001","category":"suspicious","input":"Hi, this is support. We need to confirm your billing info.","expected_risk_level":"suspicious","expected_scam_type":"possible_impersonation","expected_tactics":["fake authority","credential request","vague sender"]}
+{"id":"suspicious_sender_002","category":"suspicious","input":"This is the office. Please send your date of birth so we can update your file.","expected_risk_level":"suspicious","expected_scam_type":"possible_credential_theft","expected_tactics":["credential request","vague sender"]}
+{"id":"suspicious_sender_003","category":"suspicious","input":"Your invoice is attached. Open it and enter the requested information.","expected_risk_level":"suspicious","expected_scam_type":"unknown_attachment","expected_tactics":["attachment bait","credential request"]}
+{"id":"suspicious_sender_004","category":"suspicious","input":"We tried reaching you. Use this temporary number from now on.","expected_risk_level":"suspicious","expected_scam_type":"unknown_new_channel","expected_tactics":["new channel","vague sender"]}
+{"id":"suspicious_sender_005","category":"suspicious","input":"I am from payroll. Confirm your direct deposit account today.","expected_risk_level":"suspicious","expected_scam_type":"possible_payroll_scam","expected_tactics":["fake authority","urgency","credential request"]}
+{"id":"suspicious_market_001","category":"suspicious","input":"I want to buy your item. I will send extra money and my mover will pick it up.","expected_risk_level":"suspicious","expected_scam_type":"possible_overpayment","expected_tactics":["overpayment","marketplace"]}
+{"id":"suspicious_market_002","category":"suspicious","input":"Can you verify you are real by sending me the code I just texted you?","expected_risk_level":"suspicious","expected_scam_type":"account_takeover","expected_tactics":["credential request","verification code"]}
+{"id":"suspicious_market_003","category":"suspicious","input":"I only use this escrow service for marketplace purchases: https://safe-escrow-pay.example","expected_risk_level":"suspicious","expected_scam_type":"fake_escrow","expected_tactics":["suspicious link","marketplace","payment pressure"]}
+{"id":"suspicious_market_004","category":"suspicious","input":"The courier needs a small insurance payment before pickup. I will reimburse you.","expected_risk_level":"suspicious","expected_scam_type":"advance_fee","expected_tactics":["payment request","marketplace","advance fee"]}
+{"id":"suspicious_market_005","category":"suspicious","input":"I am out of town but my assistant will handle payment. Please email your bank name.","expected_risk_level":"suspicious","expected_scam_type":"possible_overpayment","expected_tactics":["marketplace","credential request","third party"]}
+{"id":"needs_check_bank_001","category":"needs_check","input":"Bank fraud alert: Did you attempt a $249.00 purchase at TARGET? Reply YES or NO.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_alert","expected_tactics":["verification needed"]}
+{"id":"needs_check_bank_002","category":"needs_check","input":"Your bank card ending 1234 was used for $18.42 at Grocery Store. If this was not you, call the number on your card.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_alert","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_bank_003","category":"needs_check","input":"Credit union alert: new login from a browser. Open your banking app directly if this was not you.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_alert","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_bank_004","category":"needs_check","input":"Your statement is ready. Log in through the official app to view it.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_alert","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_bank_005","category":"needs_check","input":"A transfer is scheduled for tomorrow. If you did not schedule this, contact customer service using the number on your card.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_alert","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_delivery_001","category":"needs_check","input":"UPS update: your package is out for delivery today. Track it in the UPS app.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_delivery","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_delivery_002","category":"needs_check","input":"FedEx delivery exception: weather delay reported. Check status on the official FedEx site.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_delivery","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_delivery_003","category":"needs_check","input":"Your pharmacy order shipped. Open your pharmacy account if you need details.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_delivery","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_delivery_004","category":"needs_check","input":"A package requiring signature is scheduled for tomorrow. No reply needed.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_delivery","expected_tactics":["verification needed"]}
+{"id":"needs_check_delivery_005","category":"needs_check","input":"Your store pickup is ready. Bring your ID and order confirmation to the service desk.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_delivery","expected_tactics":["verification needed"]}
+{"id":"needs_check_medical_001","category":"needs_check","input":"Reminder: dental appointment tomorrow at 2 PM. Call the office number you already have if you need to reschedule.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_appointment","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_medical_002","category":"needs_check","input":"Your lab results are available in the patient portal. Log in through the clinic website.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_health","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_medical_003","category":"needs_check","input":"Insurance notice: a claim was processed. Review it in your official member portal.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_health","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_account_001","category":"needs_check","input":"Your password was changed. If this was not you, open the app directly and review account security.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_security","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_account_002","category":"needs_check","input":"Your verification code is 482910. Do not share this code with anyone.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_security","expected_tactics":["verification needed","code warning"]}
+{"id":"needs_check_account_003","category":"needs_check","input":"New device added to your account. Visit your account settings through the official app if this was not you.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_security","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_school_001","category":"needs_check","input":"School reminder: permission forms are due Friday. Check the parent portal for details.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_notice","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_school_002","category":"needs_check","input":"Library notice: your borrowed item is overdue. Log in to your library account to renew.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_notice","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_billing_001","category":"needs_check","input":"Your utility bill is ready. No payment is due until next week. View it through your normal account.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_billing","expected_tactics":["verification needed","trusted route"]}
+{"id":"needs_check_billing_002","category":"needs_check","input":"Payment reminder from your subscription service. If unsure, open the official app instead of using message links.","expected_risk_level":"needs_check","expected_scam_type":"possible_legitimate_billing","expected_tactics":["verification needed","trusted route"]}
+{"id":"safe_personal_001","category":"safe","input":"Running ten minutes late. See you at the restaurant soon.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_personal_002","category":"safe","input":"Can you bring milk on the way home?","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_personal_003","category":"safe","input":"Happy birthday! Hope you have a great day.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_personal_004","category":"safe","input":"The meeting moved to 3 PM. Same room as before.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_personal_005","category":"safe","input":"Dinner is at our house Saturday. No need to bring anything.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_work_001","category":"safe","input":"Please review the agenda before our team call tomorrow.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_work_002","category":"safe","input":"The report draft is in the shared project folder.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_work_003","category":"safe","input":"I moved the deadline to Monday so everyone has more time.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_work_004","category":"safe","input":"Thanks for sending the notes. I added two comments.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_work_005","category":"safe","input":"Reminder: the office is closed Friday for the holiday.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_marketing_001","category":"safe","input":"Weekend sale: 20 percent off selected items in store.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_marketing_002","category":"safe","input":"Your coffee shop rewards balance updated after your last visit.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_marketing_003","category":"safe","input":"New class schedule posted for next month.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_marketing_004","category":"safe","input":"Your local bookstore has your preorder ready at the counter.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}
+{"id":"safe_marketing_005","category":"safe","input":"Thanks for your purchase. Your receipt is available in your account.","expected_risk_level":"safe","expected_scam_type":"none","expected_tactics":[]}

tests/test_eval_dataset.py

@@ -0,0 +1,31 @@
+import json
+from pathlib import Path
+
+from jawbreaker.schema import RISK_LEVELS
+
+
+def test_eval_dataset_has_100_unique_cases() -> None:
+    rows = [
+        json.loads(line)
+        for line in Path("eval/scam_eval.jsonl").read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+    assert len(rows) == 100
+    assert len({row["id"] for row in rows}) == 100
+
+
+def test_eval_dataset_required_fields_and_risk_levels() -> None:
+    required = {"id", "category", "input", "expected_risk_level", "expected_scam_type", "expected_tactics"}
+    rows = [
+        json.loads(line)
+        for line in Path("eval/scam_eval.jsonl").read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+    for row in rows:
+        assert required <= set(row)
+        assert row["expected_risk_level"] in RISK_LEVELS
+        assert isinstance(row["expected_tactics"], list)
+        assert ".example" in row["input"] or "http" not in row["input"].lower()
+