tests/test_evaluation_runner.py

import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

from evaluation_runner import (
    AnswerCache,
    EvaluationClient,
    EvaluationError,
    RunnerSettings,
    safe_filename,
)


class AnswerCacheTests(unittest.TestCase):
    def test_round_trip_and_question_hash(self) -> None:
        question = {"task_id": "abc", "question": "What is 2 + 2?"}
        with tempfile.TemporaryDirectory() as directory:
            path = Path(directory) / "answers.json"
            cache = AnswerCache(path)
            cache.record(question, "4", "test-agent", None)

            reloaded = AnswerCache(path)
            self.assertEqual(reloaded.get_valid(question), "4")
            changed = {"task_id": "abc", "question": "What is 3 + 3?"}
            self.assertIsNone(reloaded.get_valid(changed))
            self.assertEqual(json.loads(path.read_text())["version"], 1)


class PathSafetyTests(unittest.TestCase):
    def test_filename_discards_parent_directories(self) -> None:
        self.assertEqual(safe_filename("../../secret.py"), "secret.py")
        self.assertEqual(safe_filename(r"..\..\sheet data.xlsx"), "sheet_data.xlsx")

    def test_filename_rejects_empty_component(self) -> None:
        with self.assertRaises(EvaluationError):
            safe_filename("..")


class GaiaFallbackTests(unittest.TestCase):
    def test_official_dataset_file_is_copied_to_private_task_cache(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            downloaded = root / "hub-file.mp3"
            downloaded.write_bytes(b"test audio")
            destination = root / "attachments" / "task.mp3"
            settings = RunnerSettings(
                api_url="https://example.test",
                username="tester",
                space_id="tester/space",
                local_dir=root,
                gaia_repo_id="gaia-benchmark/GAIA",
                gaia_data_dir="2023/validation",
            )
            client = EvaluationClient(settings)

            with patch(
                "huggingface_hub.hf_hub_download", return_value=str(downloaded)
            ) as hub_download:
                result = client._download_gaia_attachment("task.mp3", destination)

            self.assertEqual(result, destination)
            self.assertEqual(destination.read_bytes(), b"test audio")
            self.assertEqual(
                hub_download.call_args.kwargs["filename"],
                "2023/validation/task.mp3",
            )


if __name__ == "__main__":
    unittest.main()