Spaces:
Sleeping
Sleeping
| """ | |
| OpenEnv spec compliance tests — validates openv.yaml structure without | |
| requiring the openenv package to be installed. | |
| """ | |
| import importlib | |
| from pathlib import Path | |
| import pytest | |
| import yaml | |
| SPEC_PATH = Path(__file__).parent.parent / "grid_env" / "openv.yaml" | |
| REQUIRED_TOP_LEVEL = {"spec_version", "name", "entrypoint", "models", "methods", "tasks"} | |
| REQUIRED_MODELS = {"action", "observation", "state"} | |
| REQUIRED_TASK_FIELDS = {"id", "grader"} | |
| def spec(): | |
| assert SPEC_PATH.exists(), f"openv.yaml not found at {SPEC_PATH}" | |
| with SPEC_PATH.open() as f: | |
| return yaml.safe_load(f) | |
| def test_yaml_parses_successfully(spec): | |
| assert spec is not None | |
| assert isinstance(spec, dict) | |
| def test_required_top_level_fields_present(spec): | |
| missing = REQUIRED_TOP_LEVEL - set(spec.keys()) | |
| assert not missing, f"Missing top-level fields: {missing}" | |
| def test_spec_version_is_string(spec): | |
| assert isinstance(spec["spec_version"], str) | |
| assert spec["spec_version"].strip() != "" | |
| def test_name_is_non_empty_string(spec): | |
| assert isinstance(spec["name"], str) | |
| assert spec["name"].strip() != "" | |
| def test_entrypoint_format(spec): | |
| """Entrypoint should be 'module:ClassName' style.""" | |
| entrypoint = spec["entrypoint"] | |
| assert isinstance(entrypoint, str) | |
| assert ":" in entrypoint, "entrypoint must be 'module:ClassName'" | |
| def test_entrypoint_module_is_importable(spec): | |
| module_path, _ = spec["entrypoint"].split(":", 1) | |
| try: | |
| importlib.import_module(module_path) | |
| except ImportError as exc: | |
| pytest.fail(f"Entrypoint module '{module_path}' is not importable: {exc}") | |
| def test_models_has_required_keys(spec): | |
| models = spec.get("models", {}) | |
| missing = REQUIRED_MODELS - set(models.keys()) | |
| assert not missing, f"Missing model keys: {missing}" | |
| def test_methods_contains_reset_step_state(spec): | |
| methods = set(spec.get("methods", [])) | |
| assert {"reset", "step", "state"}.issubset(methods), \ | |
| f"methods must include reset, step, state. Got: {methods}" | |
| def test_tasks_list_has_at_least_three_entries(spec): | |
| tasks = spec.get("tasks", []) | |
| assert len(tasks) >= 3, f"Expected ≥3 tasks, got {len(tasks)}" | |
| def test_each_task_has_required_fields(spec): | |
| for task in spec.get("tasks", []): | |
| missing = REQUIRED_TASK_FIELDS - set(task.keys()) | |
| assert not missing, f"Task {task} missing fields: {missing}" | |
| def test_task_ids_are_unique(spec): | |
| ids = [t["id"] for t in spec.get("tasks", [])] | |
| assert len(ids) == len(set(ids)), "Duplicate task IDs found in spec" | |
| def test_grader_references_are_importable(spec): | |
| """Each grader in the spec should resolve to a callable.""" | |
| for task in spec.get("tasks", []): | |
| grader_ref = task.get("grader", "") | |
| assert ":" in grader_ref, f"Grader '{grader_ref}' is not 'module:fn' format" | |
| mod_path, fn_name = grader_ref.split(":", 1) | |
| try: | |
| mod = importlib.import_module(mod_path) | |
| except ImportError as exc: | |
| pytest.fail(f"Cannot import grader module '{mod_path}': {exc}") | |
| fn = getattr(mod, fn_name, None) | |
| assert callable(fn), f"'{fn_name}' in '{mod_path}' is not callable" | |
| def test_baseline_section_present(spec): | |
| assert "baseline" in spec | |
| baseline = spec["baseline"] | |
| assert "runner" in baseline and "seed" in baseline | |