""" OpenEnv spec compliance tests — validates openv.yaml structure without requiring the openenv package to be installed. """ import importlib from pathlib import Path import pytest import yaml SPEC_PATH = Path(__file__).parent.parent / "grid_env" / "openv.yaml" REQUIRED_TOP_LEVEL = {"spec_version", "name", "entrypoint", "models", "methods", "tasks"} REQUIRED_MODELS = {"action", "observation", "state"} REQUIRED_TASK_FIELDS = {"id", "grader"} @pytest.fixture(scope="module") def spec(): assert SPEC_PATH.exists(), f"openv.yaml not found at {SPEC_PATH}" with SPEC_PATH.open() as f: return yaml.safe_load(f) def test_yaml_parses_successfully(spec): assert spec is not None assert isinstance(spec, dict) def test_required_top_level_fields_present(spec): missing = REQUIRED_TOP_LEVEL - set(spec.keys()) assert not missing, f"Missing top-level fields: {missing}" def test_spec_version_is_string(spec): assert isinstance(spec["spec_version"], str) assert spec["spec_version"].strip() != "" def test_name_is_non_empty_string(spec): assert isinstance(spec["name"], str) assert spec["name"].strip() != "" def test_entrypoint_format(spec): """Entrypoint should be 'module:ClassName' style.""" entrypoint = spec["entrypoint"] assert isinstance(entrypoint, str) assert ":" in entrypoint, "entrypoint must be 'module:ClassName'" def test_entrypoint_module_is_importable(spec): module_path, _ = spec["entrypoint"].split(":", 1) try: importlib.import_module(module_path) except ImportError as exc: pytest.fail(f"Entrypoint module '{module_path}' is not importable: {exc}") def test_models_has_required_keys(spec): models = spec.get("models", {}) missing = REQUIRED_MODELS - set(models.keys()) assert not missing, f"Missing model keys: {missing}" def test_methods_contains_reset_step_state(spec): methods = set(spec.get("methods", [])) assert {"reset", "step", "state"}.issubset(methods), \ f"methods must include reset, step, state. Got: {methods}" def test_tasks_list_has_at_least_three_entries(spec): tasks = spec.get("tasks", []) assert len(tasks) >= 3, f"Expected ≥3 tasks, got {len(tasks)}" def test_each_task_has_required_fields(spec): for task in spec.get("tasks", []): missing = REQUIRED_TASK_FIELDS - set(task.keys()) assert not missing, f"Task {task} missing fields: {missing}" def test_task_ids_are_unique(spec): ids = [t["id"] for t in spec.get("tasks", [])] assert len(ids) == len(set(ids)), "Duplicate task IDs found in spec" def test_grader_references_are_importable(spec): """Each grader in the spec should resolve to a callable.""" for task in spec.get("tasks", []): grader_ref = task.get("grader", "") assert ":" in grader_ref, f"Grader '{grader_ref}' is not 'module:fn' format" mod_path, fn_name = grader_ref.split(":", 1) try: mod = importlib.import_module(mod_path) except ImportError as exc: pytest.fail(f"Cannot import grader module '{mod_path}': {exc}") fn = getattr(mod, fn_name, None) assert callable(fn), f"'{fn_name}' in '{mod_path}' is not callable" def test_baseline_section_present(spec): assert "baseline" in spec baseline = spec["baseline"] assert "runner" in baseline and "seed" in baseline