"""Answer validation: turn (answer, check-spec) into a Result(verdict, hint). verdict in {"correct", "partial", "wrong"}. `hint` is filled only by the llm_judge (it knows which rubric items were missed); deterministic strategies leave it None and the orchestrator falls back to the question's concept_brief. keyword/ast are PURE (answer, spec) -> Result, so they stay module functions. The Validator class exists for the one stateful strategy -- llm_judge needs a judge backend, injected once at construction (no module globals). """ from __future__ import annotations import ast from typing import NamedTuple class Result(NamedTuple): verdict: str # "correct" | "partial" | "wrong" hint: str | None = None # judge-provided; None for deterministic strategies # --- pure strategies (no state) ------------------------------------------- def _keyword(answer: str, spec: dict) -> Result: a = answer.lower() if any(k.lower() in a for k in spec.get("correct_any", [])): return Result("correct") if any(k.lower() in a for k in spec.get("partial_any", [])): return Result("partial") return Result("wrong") def _called_names(code: str) -> set[str]: """Every function/method name called anywhere in the code.""" try: tree = ast.parse(code) except SyntaxError: return set() names = set() for node in ast.walk(tree): if isinstance(node, ast.Call): func = node.func if isinstance(func, ast.Attribute): # obj.encode(...) names.add(func.attr) elif isinstance(func, ast.Name): # tokenizer(...) names.add(func.id) return names def _ast(answer: str, spec: dict) -> Result: wanted = set(spec.get("must_call_any", [])) return Result("correct") if _called_names(answer) & wanted else Result("wrong") # --- the dispatcher (holds the one stateful strategy) --------------------- class Validator: def __init__(self, judge=None): # judge(rubric: list[str], answer: str) -> (verdict, hint); None = safe stub self.judge = judge def validate(self, answer: str, check: dict) -> Result: """Dispatch to the strategy named in the quest's `check` block.""" strategy, spec = check["strategy"], check.get("spec", {}) if strategy == "keyword": return _keyword(answer, spec) if strategy == "ast": return _ast(answer, spec) if strategy == "llm_judge": return self._llm_judge(answer, spec) raise ValueError(f"unknown strategy: {strategy!r}") def _llm_judge(self, answer: str, spec: dict) -> Result: if self.judge is None: # no judge wired -> safe stub return Result("correct" if answer.strip() else "wrong") verdict, hint = self.judge(spec.get("rubric", []), answer) return Result(verdict, hint)