AgentnessBench / tests /runtime /test_session.py
irregular6612's picture
feat(cp3): guard cut-replay termination + cover survived/eliminated/cut-frame contracts
81b3758
Raw
History Blame
3.92 kB
from proteus.providers import FakeProvider
from proteus.agents import VanillaAgent
from proteus.runtime.session import SessionRunner
from proteus.runtime.trace import SessionTrace
def _agent(responses):
return VanillaAgent(FakeProvider(responses=responses))
def test_optimal_player_survives_and_scores_full_motive_reading():
# At the EASY handover the motive-congruent action is "up". An agent that
# always plays "up"... will move up the open column away from the predator.
# Whatever the realized states, the runner scores each turn against the
# live optimal answer key. Here we script an agent that always says "up".
agent = _agent(["ACTION: up"]) # FakeProvider repeats the last response
runner = SessionRunner(
"predator_evade", agent, seed=42, play_turns=10, use_probe=False,
)
trace = runner.run()
assert isinstance(trace, SessionTrace)
assert trace.scenario == "predator_evade"
assert trace.cut_frames # Cut history captured
assert len(trace.turns) >= 1
# First played turn: the handover, where motive=up, habit=left (diagnostic).
first = trace.turns[0]
assert first.is_diagnostic is True
assert first.motive_action == "up"
assert first.habit_action == "left"
assert first.action == "up"
assert first.was_congruent is True
assert "motive_reading_accuracy" in trace.metrics
def test_habit_player_diverges_on_first_diagnostic_turn():
# An agent that always plays "left" follows inertia into the dead-end.
agent = _agent(["ACTION: left"])
runner = SessionRunner(
"predator_evade", agent, seed=42, play_turns=10, use_probe=False,
)
trace = runner.run()
first = trace.turns[0]
assert first.action == "left"
assert first.was_congruent is False
assert trace.metrics["first_divergence_turn"] == 1.0
def test_probe_recorded_when_enabled():
agent = _agent(["the predator is to my east; I should go up\nACTION: up"])
runner = SessionRunner(
"predator_evade", agent, seed=42, play_turns=3, use_probe=True,
)
trace = runner.run()
assert trace.turns[0].probe_q # a question was asked
assert trace.turns[0].probe_a # an answer was recorded
def test_session_is_deterministic_for_same_inputs():
t1 = SessionRunner("predator_evade", _agent(["ACTION: up"]), seed=42,
play_turns=5, use_probe=False).run()
t2 = SessionRunner("predator_evade", _agent(["ACTION: up"]), seed=42,
play_turns=5, use_probe=False).run()
# Same scripted agent + same seed -> identical realized trajectory.
assert [t.focal_pos for t in t1.turns] == [t.focal_pos for t in t2.turns]
assert t1.metrics == t2.metrics
def test_short_budget_yields_survived_outcome():
# With a tiny budget the step count is exhausted (without capture) right
# after the played turns, so the engine fires `survived`.
agent = _agent(["ACTION: up"])
trace = SessionRunner(
"predator_evade", agent, seed=42, play_turns=1, use_probe=False,
).run()
assert trace.outcome == "survived"
assert trace.turns[-1].reward == 50.0 # _REWARD_SURVIVED
def test_eliminated_outcome_is_explicit_and_terminal():
# The habit player ("left") walks into the dead-end and is caught.
agent = _agent(["ACTION: left"])
trace = SessionRunner(
"predator_evade", agent, seed=42, play_turns=15, use_probe=False,
).run()
assert trace.outcome == "eliminated"
assert len(trace.turns) <= 15 # stopped at/under budget
assert trace.turns[-1].reward == -50.0 # _REWARD_CAPTURED
def test_cut_frames_count_matches_cut_length_plus_one():
agent = _agent(["ACTION: up"])
trace = SessionRunner(
"predator_evade", agent, seed=42, play_turns=5, use_probe=False,
).run()
# EASY cut_length is 2 -> initial frame + 2 step frames = 3.
assert len(trace.cut_frames) == 3