File size: 1,187 Bytes
490f3fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | from __future__ import annotations
import ast
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
ANSWER_RUNNER = (
ROOT
/ "src"
/ "tmcra_benchmark"
/ "legacy"
/ "run_tmcra_v4_gpt54_answers.py"
)
def test_answer_runner_rejects_evaluation_only_fields() -> None:
tree = ast.parse(ANSWER_RUNNER.read_text(encoding="utf-8"), filename=str(ANSWER_RUNNER))
functions = {
node.name: node
for node in tree.body
if isinstance(node, ast.FunctionDef)
}
rejection = functions["reject_evaluation_fields"]
literals = {
node.value
for node in ast.walk(tree)
if isinstance(node, ast.Constant) and isinstance(node.value, str)
}
assert "gold_answer" in literals
assert "answer_session_ids" in literals
assert any(
isinstance(node, ast.Raise)
for node in ast.walk(rejection)
)
def test_answer_output_does_not_persist_gold_or_gold_scores() -> None:
source = ANSWER_RUNNER.read_text(encoding="utf-8")
assert 'row.get("gold_answer")' not in source
assert '"deterministic_scores"' not in source
assert 'row.get("answer_session_ids")' not in source
|