tests/test_evaluation.py

from pathlib import Path

import pytest

from cache import ResultStore
from evaluation import run_evaluation, submission_answers


class FakeClient:
    def get_questions(self):
        return [
            {"task_id": "ok", "question": "q", "file_name": ""},
            {"task_id": "bad", "question": "q", "file_name": "x.py"},
        ]

    def download_attachment(self, task):
        if task["task_id"] == "bad":
            raise FileNotFoundError("missing")


class FakeAgent:
    class settings:
        worker_limit = 2

    def cached_answer(self, task):
        return None

    def solve_detailed(self, task, attachment, *, force=False):
        from agent import SolveResult

        return SolveResult("answer", "general_reasoning")


def test_per_task_isolation_and_submission_guard(tmp_path):
    results = run_evaluation(
        FakeClient(), FakeAgent(), ResultStore(tmp_path / "run.json")
    )
    assert [row["status"] for row in results] == ["ok", "agent_error"]
    with pytest.raises(ValueError, match="incomplete"):
        submission_answers(results, expected_count=None)


def test_official_answer_shape():
    payload = submission_answers(
        [{"task_id": "abc", "answer": "42", "status": "ok"}], expected_count=None
    )
    assert payload == [{"task_id": "abc", "submitted_answer": "42"}]


def test_submission_requires_exactly_twenty_unique_nonempty_answers():
    rows = [
        {"task_id": str(index), "answer": str(index), "status": "ok"}
        for index in range(20)
    ]
    assert len(submission_answers(rows)) == 20
    with pytest.raises(ValueError, match="exactly 20"):
        submission_answers(rows[:-1])
    rows[-1]["answer"] = ""
    with pytest.raises(ValueError, match="empty"):
        submission_answers(rows)


def test_dry_run_modules_have_no_submission_call():
    root = Path(__file__).resolve().parents[1]
    for filename in ("evaluation.py", "run_local_eval.py"):
        source = (root / filename).read_text(encoding="utf-8")
        assert "/submit" not in source
        assert ".submit_answers(" not in source