Spaces:
Running
Running
| """Answer validation: turn (answer, check-spec) into a Result(verdict, hint). | |
| verdict in {"correct", "partial", "wrong"}. `hint` is filled only by the | |
| llm_judge (it knows which rubric items were missed); deterministic strategies | |
| leave it None and the orchestrator falls back to the question's concept_brief. | |
| keyword/ast are PURE (answer, spec) -> Result, so they stay module functions. | |
| The Validator class exists for the one stateful strategy -- llm_judge needs a | |
| judge backend, injected once at construction (no module globals). | |
| """ | |
| from __future__ import annotations | |
| import ast | |
| from typing import NamedTuple | |
| class Result(NamedTuple): | |
| verdict: str # "correct" | "partial" | "wrong" | |
| hint: str | None = None # judge-provided; None for deterministic strategies | |
| # --- pure strategies (no state) ------------------------------------------- | |
| def _keyword(answer: str, spec: dict) -> Result: | |
| a = answer.lower() | |
| if any(k.lower() in a for k in spec.get("correct_any", [])): | |
| return Result("correct") | |
| if any(k.lower() in a for k in spec.get("partial_any", [])): | |
| return Result("partial") | |
| return Result("wrong") | |
| def _called_names(code: str) -> set[str]: | |
| """Every function/method name called anywhere in the code.""" | |
| try: | |
| tree = ast.parse(code) | |
| except SyntaxError: | |
| return set() | |
| names = set() | |
| for node in ast.walk(tree): | |
| if isinstance(node, ast.Call): | |
| func = node.func | |
| if isinstance(func, ast.Attribute): # obj.encode(...) | |
| names.add(func.attr) | |
| elif isinstance(func, ast.Name): # tokenizer(...) | |
| names.add(func.id) | |
| return names | |
| def _ast(answer: str, spec: dict) -> Result: | |
| wanted = set(spec.get("must_call_any", [])) | |
| return Result("correct") if _called_names(answer) & wanted else Result("wrong") | |
| # --- the dispatcher (holds the one stateful strategy) --------------------- | |
| class Validator: | |
| def __init__(self, judge=None): | |
| # judge(rubric: list[str], answer: str) -> (verdict, hint); None = safe stub | |
| self.judge = judge | |
| def validate(self, answer: str, check: dict) -> Result: | |
| """Dispatch to the strategy named in the quest's `check` block.""" | |
| strategy, spec = check["strategy"], check.get("spec", {}) | |
| if strategy == "keyword": | |
| return _keyword(answer, spec) | |
| if strategy == "ast": | |
| return _ast(answer, spec) | |
| if strategy == "llm_judge": | |
| return self._llm_judge(answer, spec) | |
| raise ValueError(f"unknown strategy: {strategy!r}") | |
| def _llm_judge(self, answer: str, spec: dict) -> Result: | |
| if self.judge is None: # no judge wired -> safe stub | |
| return Result("correct" if answer.strip() else "wrong") | |
| verdict, hint = self.judge(spec.get("rubric", []), answer) | |
| return Result(verdict, hint) | |