tests/test_evaluation_config.py

import importlib.util
import unittest
from pathlib import Path


def load_module(path):
    spec = importlib.util.spec_from_file_location(path.stem, path)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


class EvaluationConfigTest(unittest.TestCase):
    def test_context_baseline_lists_are_deduplicated(self):
        module = load_module(Path("evaluation/context_eval_baselines.py"))
        self.assertEqual(len(module.METHODS), len(set(module.METHODS)))
        self.assertEqual(len(module.LLM_METHODS), len(set(module.LLM_METHODS)))

    def test_to_pred_requires_explicit_task(self):
        module = load_module(Path("evaluation/tools/to_pred.py"))
        self.assertIn("interactable", module.TASKS)
        self.assertIn("all", module.build_parser()._option_string_actions["--task"].choices)


if __name__ == "__main__":
    unittest.main()