query/validate100.py

"""Step 7 — acceptance battery: run all 100 CEO questions through the brain and score them.

A question PASSES when the answer is grounded and on-topic:
  - non-empty answer of reasonable length,
  - NOT the generic 'couldn't find' fallback,
  - has at least one citation OR is a meta/limits answer (which legitimately has none),
  - resolves to a concrete target (activity/table/sp/api) OR a meta/doc intent.

Reports per-category and overall, and lists every weak/failed question so we can fix them.
"""
from __future__ import annotations

import json

import config
from query.ask import answer

FALLBACK_MARKERS = ("I couldn't find", "couldn't find that in the PO brain")
NO_CITE_OK = {"META_LIMITS"}


def score_one(q: str) -> tuple[bool, str]:
    # Validate the GROUNDED deterministic substrate (the LLM only rephrases it).
    r = answer(q, use_llm=False)
    ans = (r.get("answer") or "").strip()
    intent = r.get("intent", "")
    cites = r.get("citations", [])
    target = r.get("target")
    if len(ans) < 40:
        return False, f"thin answer ({len(ans)} chars)"
    if any(m in ans for m in FALLBACK_MARKERS):
        return False, "generic fallback"
    if not cites and intent not in NO_CITE_OK:
        return False, f"no citation (intent={intent})"
    if not target and intent not in ("META_LIMITS", "META_JOURNEY", "WHAT_IS", "COMPARE"):
        return False, f"no target (intent={intent})"
    return True, intent


def main():
    cats = json.loads(config.QUESTIONS_JSON.read_text())
    total = passed = 0
    weak = []
    print(f"{'CATEGORY':40s}  PASS")
    print("-" * 52)
    for block in cats:
        cat = block["category"]
        cp = ct = 0
        for q in block["questions"]:
            ok, why = score_one(q)
            ct += 1; total += 1
            if ok:
                cp += 1; passed += 1
            else:
                weak.append((cat, q, why))
        print(f"{cat:40s}  {cp}/{ct}")
    print("-" * 52)
    print(f"{'OVERALL':40s}  {passed}/{total}  ({100*passed//total}%)")
    if weak:
        print(f"\nWEAK / FAILED ({len(weak)}):")
        for cat, q, why in weak:
            print(f"  [{why}] {q}")
    return passed, total


if __name__ == "__main__":
    main()