tests/test_environment.py

"""
Tests for the API Contract Validator Environment.

Run from the api_contract_validator/ directory:
    pytest tests/ -v
"""

import sys
import os

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import pytest
from server.environment import ValidatorEnvironment
from server.spec_generator import generate_scenario_for_task, AVAILABLE_TASKS
from models import ValidatorAction


@pytest.fixture
def env():
    """Fresh environment for each test."""
    return ValidatorEnvironment()


# ── Task structure ─────────────────────────────────────────────────────────


def test_six_tasks_registered():
    assert len(AVAILABLE_TASKS) == 6
    expected = {
        "find_type_mismatches",
        "validate_nested_objects",
        "detect_breaking_changes",
        "validate_response_schema",
        "validate_cross_field_constraints",
        "validate_auth_request",
    }
    assert set(AVAILABLE_TASKS) == expected


def test_all_tasks_have_violations():
    for task_name in AVAILABLE_TASKS:
        scenario = generate_scenario_for_task(task_name)
        assert len(scenario.violations) >= 4, (
            f"{task_name} has only {len(scenario.violations)} violations"
        )
        assert scenario.max_steps >= len(scenario.violations), (
            f"{task_name}: max_steps({scenario.max_steps}) < violations({len(scenario.violations)})"
        )


# ── Reset behaviour ────────────────────────────────────────────────────────


def test_all_tasks_reset_cleanly(env):
    for task_name in AVAILABLE_TASKS:
        obs = env.reset(task_name=task_name)
        assert obs.task_name == task_name
        assert obs.done is False
        assert obs.reward == 0.0
        assert obs.violations_found == []
        assert obs.violations_remaining > 0


# ── Correct violation reward ───────────────────────────────────────────────


def test_correct_violation_gives_plus_one(env):
    scenario = generate_scenario_for_task("find_type_mismatches")
    env.reset(task_name="find_type_mismatches")

    first = scenario.violations[0]
    action = ValidatorAction(
        field_path=first.field_path,
        violation_type=first.violation_type,
        description="test",
    )
    result = env.step(action)
    assert result.reward == 1.0
    assert len(result.violations_found) == 1


# ── False positive penalty ─────────────────────────────────────────────────


def test_false_positive_gives_negative_reward(env):
    env.reset(task_name="find_type_mismatches")
    action = ValidatorAction(
        field_path="nonexistent_field_xyz_abc",
        violation_type="type_mismatch",
        description="fabricated",
    )
    result = env.step(action)
    assert result.reward == pytest.approx(-0.3)


# ── Duplicate penalty ──────────────────────────────────────────────────────


def test_duplicate_gives_small_penalty(env):
    scenario = generate_scenario_for_task("find_type_mismatches")
    env.reset(task_name="find_type_mismatches")

    first = scenario.violations[0]
    action = ValidatorAction(
        field_path=first.field_path,
        violation_type=first.violation_type,
        description="test",
    )
    result1 = env.step(action)
    assert result1.reward == 1.0

    result2 = env.step(action)  # duplicate
    assert result2.reward == pytest.approx(-0.1)


# ── DONE signal ────────────────────────────────────────────────────────────


def test_done_signal_ends_episode(env):
    env.reset(task_name="find_type_mismatches")
    action = ValidatorAction(field_path="DONE", violation_type="", description="")
    result = env.step(action)
    assert result.done is True
    assert result.reward >= 0.0


# ── HINT mechanic ──────────────────────────────────────────────────────────


def test_hint_costs_half_point(env):
    env.reset(task_name="find_type_mismatches")
    action = ValidatorAction(field_path="HINT", violation_type="", description="")
    result = env.step(action)
    assert result.reward == pytest.approx(-0.5)
    assert "Hint:" in result.feedback
    assert result.done is False


# ── Proximity reward ───────────────────────────────────────────────────────


def test_proximity_reward_for_correct_path_wrong_type(env):
    scenario = generate_scenario_for_task("find_type_mismatches")
    env.reset(task_name="find_type_mismatches")

    first = scenario.violations[0]
    action = ValidatorAction(
        field_path=first.field_path,
        violation_type="extra_field",  # wrong type on purpose
        description="proximity test",
    )
    result = env.step(action)
    assert result.reward == pytest.approx(0.3)


# ── Seed reproducibility ───────────────────────────────────────────────────


def test_seed_gives_same_scenario():
    for task_name in AVAILABLE_TASKS:
        s1 = generate_scenario_for_task(task_name, seed=42)
        s2 = generate_scenario_for_task(task_name, seed=42)
        assert [v.field_path for v in s1.violations] == [
            v.field_path for v in s2.violations
        ], f"{task_name}: seed=42 gave different results across calls"


def test_different_seeds_give_different_easy_scenarios():
    """Easy task pool should vary with different seeds."""
    paths_by_seed = set()
    for seed in range(8):
        s = generate_scenario_for_task("find_type_mismatches", seed=seed)
        key = tuple(sorted(v.field_path for v in s.violations))
        paths_by_seed.add(key)
    assert len(paths_by_seed) > 1, "Different seeds produced identical scenarios"


# ── Cross-field task ───────────────────────────────────────────────────────


def test_cross_field_task_has_seven_violations():
    scenario = generate_scenario_for_task("validate_cross_field_constraints")
    assert len(scenario.violations) == 7


def test_cross_field_violations_use_correct_type():
    scenario = generate_scenario_for_task("validate_cross_field_constraints")
    for v in scenario.violations:
        assert v.violation_type == "cross_field_constraint", (
            f"Expected cross_field_constraint, got {v.violation_type} for {v.field_path}"
        )


# ── Auth task ──────────────────────────────────────────────────────────────


def test_auth_task_has_six_violations():
    scenario = generate_scenario_for_task("validate_auth_request")
    assert len(scenario.violations) == 6


def test_auth_task_variants_differ():
    s_even = generate_scenario_for_task("validate_auth_request", seed=0)
    s_odd = generate_scenario_for_task("validate_auth_request", seed=1)
    paths_even = {v.field_path for v in s_even.violations}
    paths_odd = {v.field_path for v in s_odd.violations}
    assert paths_even != paths_odd, "Even and odd seed should give different auth scenarios"


# ── Easy pool expansion ────────────────────────────────────────────────────


def test_easy_pool_has_twelve_variants():
    from server.spec_generator import _EASY_POOL
    assert len(_EASY_POOL) == 12, f"Expected 12 pool entries, got {len(_EASY_POOL)}"


# ── Phase 2 — impact tracing ───────────────────────────────────────────────


def test_phase2_reset_returns_service_graph(env):
    obs = env.reset(task_name="trace_downstream_blast_radius", seed=1)
    assert obs.phase == "tracing"
    assert obs.total_consumers >= 3
    assert "consumers" in obs.service_graph
    assert obs.feedback


def test_phase2_perfect_trace_scores_high(env):
    env.reset(task_name="trace_downstream_blast_radius", seed=1)
    action = ValidatorAction(
        action_type="trace_impact",
        affected_services=[
            "OrdersService",
            "BillingService",
            "NotificationsService",
        ],
    )
    result = env.step(action)
    assert result.done is True
    assert result.reward > 2.0  # 3 hits @ +0.8 each
    assert env.state.score > 0.9


def test_phase2_false_flag_penalty(env):
    env.reset(task_name="trace_downstream_blast_radius", seed=1)
    action = ValidatorAction(
        action_type="trace_impact",
        affected_services=["OrdersService", "AnalyticsETL"],  # one false flag
    )
    result = env.step(action)
    assert result.done is True
    # 1 hit (+0.8) + 2 missed (-0.5 each) + 1 false (-0.4) = -0.6
    assert result.reward < 0


def test_phase2_unknown_service_treated_as_false_flag(env):
    env.reset(task_name="trace_downstream_blast_radius", seed=1)
    action = ValidatorAction(
        action_type="trace_impact",
        affected_services=["NonexistentService"],
    )
    result = env.step(action)
    assert result.done is True
    assert result.reward < 0


# ── Phase 3 — fix proposal ─────────────────────────────────────────────────


def test_phase3_reset_returns_violation_and_consumers(env):
    obs = env.reset(task_name="propose_backward_compat_fix", seed=1)
    assert obs.phase == "fix_proposal"
    assert obs.detected_violation
    assert obs.consumer_specs


def test_phase3_good_field_alias_passes_all_consumers(env):
    env.reset(task_name="propose_backward_compat_fix", seed=1)
    action = ValidatorAction(
        action_type="propose_fix",
        fix_strategy="field_alias",
        spec_patch={"aliases": {"email": "email_address"}},
        rationale="Keep old field name as alias",
    )
    result = env.step(action)
    assert result.done is True
    assert result.reward >= 2.0
    assert env.state.fix_validated is True


def test_phase3_malformed_strategy_penalty(env):
    env.reset(task_name="propose_backward_compat_fix", seed=1)
    action = ValidatorAction(
        action_type="propose_fix",
        fix_strategy="not_a_real_strategy",
        spec_patch={},
    )
    result = env.step(action)
    assert result.reward < 0
    assert env.state.fix_validated is False


def test_phase3_breaking_consumer_penalised(env):
    env.reset(task_name="propose_backward_compat_fix", seed=1)
    # dual_write but missing the new field — breaks all consumers
    action = ValidatorAction(
        action_type="propose_fix",
        fix_strategy="dual_write",
        spec_patch={"emit_fields": ["email"]},
    )
    result = env.step(action)
    assert result.reward < 0
    assert env.state.fix_validated is False


# ── Cascade — full workflow ───────────────────────────────────────────────


def test_cascade_starts_in_tracing_phase(env):
    obs = env.reset(task_name="multi_service_cascade_fix", seed=1)
    assert obs.phase == "tracing"


def test_cascade_transitions_to_fix_after_correct_trace(env):
    env.reset(task_name="multi_service_cascade_fix", seed=1)
    trace = ValidatorAction(
        action_type="trace_impact",
        affected_services=[
            "OrdersService",
            "BillingService",
            "NotificationsService",
        ],
    )
    obs = env.step(trace)
    assert obs.done is False
    assert obs.phase == "fix_proposal"

    fix = ValidatorAction(
        action_type="propose_fix",
        fix_strategy="field_alias",
        spec_patch={"aliases": {"email": "email_address"}},
    )
    obs = env.step(fix)
    assert obs.done is True
    assert env.state.fix_validated is True


# ── Determinism ────────────────────────────────────────────────────────────


def test_phase2_seed_determinism(env):
    obs1 = env.reset(task_name="trace_downstream_blast_radius", seed=1)
    obs2 = env.reset(task_name="trace_downstream_blast_radius", seed=1)
    services1 = sorted(c["name"] for c in obs1.service_graph["consumers"])
    services2 = sorted(c["name"] for c in obs2.service_graph["consumers"])
    assert services1 == services2


def test_different_seeds_pick_different_scenarios(env):
    obs_even = env.reset(task_name="trace_downstream_blast_radius", seed=0)
    obs_odd = env.reset(task_name="trace_downstream_blast_radius", seed=1)
    name_even = obs_even.service_graph["producer"]
    name_odd = obs_odd.service_graph["producer"]
    assert name_even != name_odd