File size: 2,084 Bytes
c641d5f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | from pathlib import Path
import pytest
from cache import ResultStore
from evaluation import run_evaluation, submission_answers
class FakeClient:
def get_questions(self):
return [
{"task_id": "ok", "question": "q", "file_name": ""},
{"task_id": "bad", "question": "q", "file_name": "x.py"},
]
def download_attachment(self, task):
if task["task_id"] == "bad":
raise FileNotFoundError("missing")
class FakeAgent:
class settings:
worker_limit = 2
def cached_answer(self, task):
return None
def solve_detailed(self, task, attachment, *, force=False):
from agent import SolveResult
return SolveResult("answer", "general_reasoning")
def test_per_task_isolation_and_submission_guard(tmp_path):
results = run_evaluation(
FakeClient(), FakeAgent(), ResultStore(tmp_path / "run.json")
)
assert [row["status"] for row in results] == ["ok", "agent_error"]
with pytest.raises(ValueError, match="incomplete"):
submission_answers(results, expected_count=None)
def test_official_answer_shape():
payload = submission_answers(
[{"task_id": "abc", "answer": "42", "status": "ok"}], expected_count=None
)
assert payload == [{"task_id": "abc", "submitted_answer": "42"}]
def test_submission_requires_exactly_twenty_unique_nonempty_answers():
rows = [
{"task_id": str(index), "answer": str(index), "status": "ok"}
for index in range(20)
]
assert len(submission_answers(rows)) == 20
with pytest.raises(ValueError, match="exactly 20"):
submission_answers(rows[:-1])
rows[-1]["answer"] = ""
with pytest.raises(ValueError, match="empty"):
submission_answers(rows)
def test_dry_run_modules_have_no_submission_call():
root = Path(__file__).resolve().parents[1]
for filename in ("evaluation.py", "run_local_eval.py"):
source = (root / filename).read_text(encoding="utf-8")
assert "/submit" not in source
assert ".submit_answers(" not in source
|