"""Tests for the typed Validator plan module.""" import pytest from arena.models import Criteria, ValidationCheck from arena.validator_plan import ( CommandValidationStep, RubricValidationStep, StagehandValidationStep, ValidatorContext, ValidatorPlan, validation_checks_from_user_tests, ) def test_validator_plan_from_user_tests_separates_command_and_rubric_steps() -> None: plan = ValidatorPlan.from_user_tests(["pytest -q", "ruff check"]) assert len(plan.command_steps) == 2 assert all(isinstance(step, CommandValidationStep) for step in plan.command_steps) assert plan.stagehand_steps == () assert isinstance(plan.rubric_step, RubricValidationStep) assert [step.check.command for step in plan.command_steps] == ["pytest -q", "ruff check"] def test_validator_plan_from_checks_routes_stagehand_to_typed_step() -> None: checks = [ ValidationCheck(id="cmd-1", name="tests", command="pytest -q", source="user"), ValidationCheck(id="vis-1", name="hero loads", command="https://app.example", source="stagehand"), ] plan = ValidatorPlan.from_checks(checks) assert [step.check.id for step in plan.command_steps] == ["cmd-1"] assert [step.check.id for step in plan.stagehand_steps] == ["vis-1"] assert isinstance(plan.stagehand_steps[0], StagehandValidationStep) def test_validator_plan_can_disable_rubric_step() -> None: plan = ValidatorPlan.from_user_tests(["pytest -q"], rubric_enabled=False) assert plan.rubric_step is None def test_validator_plan_carries_prompt_and_criteria_context() -> None: context = ValidatorContext(prompt="Build a CLI", criteria=(Criteria(text="pytest must pass"),)) plan = ValidatorPlan.from_user_tests(["pytest -q"], context=context) assert plan.context.prompt == "Build a CLI" assert len(plan.context.criteria) == 1 assert plan.context.criteria[0].text == "pytest must pass" def test_validator_plan_adds_gradio_launch_check_for_demo_context() -> None: context = ValidatorContext( prompt="Build a Gradio demo app", criteria=(Criteria(text="Works as a Gradio app"),), ) plan = ValidatorPlan.from_user_tests(["python -m py_compile app.py"], context=context) assert [step.check.id for step in plan.command_steps] == [ "user-test-1", "generated-gradio-smoke", ] assert "127.0.0.1:{port}" in plan.command_steps[1].check.command def test_validator_plan_checks_property_returns_ordered_checks() -> None: plan = ValidatorPlan.from_checks([ ValidationCheck(id="c-1", name="c1", command="cmd1"), ValidationCheck(id="s-1", name="s1", command="https://x", source="stagehand"), ValidationCheck(id="c-2", name="c2", command="cmd2"), ]) assert [check.id for check in plan.checks] == ["c-1", "c-2", "s-1"] def test_validation_checks_from_user_tests_assigns_stable_ids() -> None: checks = validation_checks_from_user_tests(["echo a", "echo b"]) assert [check.id for check in checks] == ["user-test-1", "user-test-2"] assert all(check.source == "user" for check in checks) def test_validator_plan_default_context_is_empty() -> None: plan = ValidatorPlan.from_user_tests(["pytest -q"]) assert plan.context == ValidatorContext() def test_validator_plan_is_immutable() -> None: plan = ValidatorPlan.from_user_tests(["pytest -q"]) with pytest.raises(Exception): plan.command_steps = ()