Spaces:
Running
Running
File size: 2,601 Bytes
2b4bd40 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from evaluation_runner import (
AnswerCache,
EvaluationClient,
EvaluationError,
RunnerSettings,
safe_filename,
)
class AnswerCacheTests(unittest.TestCase):
def test_round_trip_and_question_hash(self) -> None:
question = {"task_id": "abc", "question": "What is 2 + 2?"}
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "answers.json"
cache = AnswerCache(path)
cache.record(question, "4", "test-agent", None)
reloaded = AnswerCache(path)
self.assertEqual(reloaded.get_valid(question), "4")
changed = {"task_id": "abc", "question": "What is 3 + 3?"}
self.assertIsNone(reloaded.get_valid(changed))
self.assertEqual(json.loads(path.read_text())["version"], 1)
class PathSafetyTests(unittest.TestCase):
def test_filename_discards_parent_directories(self) -> None:
self.assertEqual(safe_filename("../../secret.py"), "secret.py")
self.assertEqual(safe_filename(r"..\..\sheet data.xlsx"), "sheet_data.xlsx")
def test_filename_rejects_empty_component(self) -> None:
with self.assertRaises(EvaluationError):
safe_filename("..")
class GaiaFallbackTests(unittest.TestCase):
def test_official_dataset_file_is_copied_to_private_task_cache(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
downloaded = root / "hub-file.mp3"
downloaded.write_bytes(b"test audio")
destination = root / "attachments" / "task.mp3"
settings = RunnerSettings(
api_url="https://example.test",
username="tester",
space_id="tester/space",
local_dir=root,
gaia_repo_id="gaia-benchmark/GAIA",
gaia_data_dir="2023/validation",
)
client = EvaluationClient(settings)
with patch(
"huggingface_hub.hf_hub_download", return_value=str(downloaded)
) as hub_download:
result = client._download_gaia_attachment("task.mp3", destination)
self.assertEqual(result, destination)
self.assertEqual(destination.read_bytes(), b"test audio")
self.assertEqual(
hub_download.call_args.kwargs["filename"],
"2023/validation/task.mp3",
)
if __name__ == "__main__":
unittest.main()
|