tests_python/test_validator_plan.py

"""Tests for the typed Validator plan module."""

import pytest

from arena.models import Criteria, ValidationCheck
from arena.validator_plan import (
    CommandValidationStep,
    RubricValidationStep,
    StagehandValidationStep,
    ValidatorContext,
    ValidatorPlan,
    validation_checks_from_user_tests,
)


def test_validator_plan_from_user_tests_separates_command_and_rubric_steps() -> None:
    plan = ValidatorPlan.from_user_tests(["pytest -q", "ruff check"])

    assert len(plan.command_steps) == 2
    assert all(isinstance(step, CommandValidationStep) for step in plan.command_steps)
    assert plan.stagehand_steps == ()
    assert isinstance(plan.rubric_step, RubricValidationStep)
    assert [step.check.command for step in plan.command_steps] == ["pytest -q", "ruff check"]


def test_validator_plan_from_checks_routes_stagehand_to_typed_step() -> None:
    checks = [
        ValidationCheck(id="cmd-1", name="tests", command="pytest -q", source="user"),
        ValidationCheck(id="vis-1", name="hero loads", command="https://app.example", source="stagehand"),
    ]

    plan = ValidatorPlan.from_checks(checks)

    assert [step.check.id for step in plan.command_steps] == ["cmd-1"]
    assert [step.check.id for step in plan.stagehand_steps] == ["vis-1"]
    assert isinstance(plan.stagehand_steps[0], StagehandValidationStep)


def test_validator_plan_can_disable_rubric_step() -> None:
    plan = ValidatorPlan.from_user_tests(["pytest -q"], rubric_enabled=False)

    assert plan.rubric_step is None


def test_validator_plan_carries_prompt_and_criteria_context() -> None:
    context = ValidatorContext(prompt="Build a CLI", criteria=(Criteria(text="pytest must pass"),))
    plan = ValidatorPlan.from_user_tests(["pytest -q"], context=context)

    assert plan.context.prompt == "Build a CLI"
    assert len(plan.context.criteria) == 1
    assert plan.context.criteria[0].text == "pytest must pass"


def test_validator_plan_adds_gradio_launch_check_for_demo_context() -> None:
    context = ValidatorContext(
        prompt="Build a Gradio demo app",
        criteria=(Criteria(text="Works as a Gradio app"),),
    )

    plan = ValidatorPlan.from_user_tests(["python -m py_compile app.py"], context=context)

    assert [step.check.id for step in plan.command_steps] == [
        "user-test-1",
        "generated-gradio-smoke",
    ]
    assert "127.0.0.1:{port}" in plan.command_steps[1].check.command


def test_validator_plan_checks_property_returns_ordered_checks() -> None:
    plan = ValidatorPlan.from_checks([
        ValidationCheck(id="c-1", name="c1", command="cmd1"),
        ValidationCheck(id="s-1", name="s1", command="https://x", source="stagehand"),
        ValidationCheck(id="c-2", name="c2", command="cmd2"),
    ])

    assert [check.id for check in plan.checks] == ["c-1", "c-2", "s-1"]


def test_validation_checks_from_user_tests_assigns_stable_ids() -> None:
    checks = validation_checks_from_user_tests(["echo a", "echo b"])

    assert [check.id for check in checks] == ["user-test-1", "user-test-2"]
    assert all(check.source == "user" for check in checks)


def test_validator_plan_default_context_is_empty() -> None:
    plan = ValidatorPlan.from_user_tests(["pytest -q"])

    assert plan.context == ValidatorContext()


def test_validator_plan_is_immutable() -> None:
    plan = ValidatorPlan.from_user_tests(["pytest -q"])

    with pytest.raises(Exception):
        plan.command_steps = ()