Spaces:
Runtime error
Runtime error
Commit ·
53ff5ab
1
Parent(s): e902aac
feat(cp6): scenario-owned survival step_reward (away-from-predator) + safety_distance
Browse files- proteus/grid/scenario.py +30 -0
- proteus/grid/scenarios/predator_evade.py +46 -0
- proteus/runtime/session.py +8 -23
- tests/grid/test_step_reward.py +57 -0
proteus/grid/scenario.py
CHANGED
|
@@ -165,6 +165,36 @@ class Scenario(ABC):
|
|
| 165 |
"""
|
| 166 |
raise NotImplementedError
|
| 167 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
@abstractmethod
|
| 169 |
def legend(self) -> dict[int, str]:
|
| 170 |
"""Return the palette-index -> symbol/meaning map for the ASCII view.
|
|
|
|
| 165 |
"""
|
| 166 |
raise NotImplementedError
|
| 167 |
|
| 168 |
+
@abstractmethod
|
| 169 |
+
def step_reward(
|
| 170 |
+
self,
|
| 171 |
+
game: MotiveGridGame,
|
| 172 |
+
action: str,
|
| 173 |
+
blocked: bool,
|
| 174 |
+
focal_before: tuple[int, int],
|
| 175 |
+
predator_before: tuple[int, int],
|
| 176 |
+
) -> float:
|
| 177 |
+
"""Return the per-turn reward for *action* under this scenario's motive.
|
| 178 |
+
|
| 179 |
+
Called AFTER the turn is fully applied (focal moved, threat advanced,
|
| 180 |
+
terminal flags set), so ``game`` holds the post-move state. The pre-move
|
| 181 |
+
focal and predator positions are passed in so distance-delta rewards can
|
| 182 |
+
isolate the agent's own move from the threat's response.
|
| 183 |
+
|
| 184 |
+
Args:
|
| 185 |
+
game: The live game in its POST-move state (focal moved, threat
|
| 186 |
+
advanced, terminal flags set).
|
| 187 |
+
action: The action the focal just committed.
|
| 188 |
+
blocked: Whether that directional move was blocked by a wall/edge
|
| 189 |
+
(no positional change).
|
| 190 |
+
focal_before: The focal's ``(x, y)`` BEFORE the move.
|
| 191 |
+
predator_before: The predator's ``(x, y)`` BEFORE the move.
|
| 192 |
+
|
| 193 |
+
Returns:
|
| 194 |
+
The per-turn reward as a float (sign/scale defined per scenario).
|
| 195 |
+
"""
|
| 196 |
+
raise NotImplementedError
|
| 197 |
+
|
| 198 |
@abstractmethod
|
| 199 |
def legend(self) -> dict[int, str]:
|
| 200 |
"""Return the palette-index -> symbol/meaning map for the ASCII view.
|
proteus/grid/scenarios/predator_evade.py
CHANGED
|
@@ -78,6 +78,11 @@ FOCAL_IDX = 1
|
|
| 78 |
PREDATOR_IDX = 2
|
| 79 |
WALL_IDX = 3
|
| 80 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
# --------------------------------------------------------------------------- #
|
| 82 |
# EASY deterministic layout.
|
| 83 |
# --------------------------------------------------------------------------- #
|
|
@@ -447,6 +452,47 @@ class PredatorEvade(Scenario):
|
|
| 447 |
if action in _DELTAS and action != "stay":
|
| 448 |
self._last_focal_move = action
|
| 449 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
# ------------------------------------------------------------------ #
|
| 451 |
# ASCII legend
|
| 452 |
# ------------------------------------------------------------------ #
|
|
|
|
| 78 |
PREDATOR_IDX = 2
|
| 79 |
WALL_IDX = 3
|
| 80 |
|
| 81 |
+
# Survival-category reward constants (moved from SessionRunner; see CP6 spec §5).
|
| 82 |
+
_REWARD_CAPTURED = -50.0
|
| 83 |
+
_REWARD_SURVIVED = 50.0
|
| 84 |
+
_REWARD_WALL_HIT = -3.0
|
| 85 |
+
|
| 86 |
# --------------------------------------------------------------------------- #
|
| 87 |
# EASY deterministic layout.
|
| 88 |
# --------------------------------------------------------------------------- #
|
|
|
|
| 452 |
if action in _DELTAS and action != "stay":
|
| 453 |
self._last_focal_move = action
|
| 454 |
|
| 455 |
+
def step_reward(
|
| 456 |
+
self,
|
| 457 |
+
game: MotiveGridGame,
|
| 458 |
+
action: str,
|
| 459 |
+
blocked: bool,
|
| 460 |
+
focal_before: tuple[int, int],
|
| 461 |
+
predator_before: tuple[int, int],
|
| 462 |
+
) -> float:
|
| 463 |
+
"""Survival reward: positive for moving away from the predator.
|
| 464 |
+
|
| 465 |
+
Terminal states dominate (capture / survival). Otherwise a blocked move
|
| 466 |
+
is penalised, and a legal move is scored by the change in BFS distance
|
| 467 |
+
from the focal to the predator's PRE-move cell (post-move focal cell vs
|
| 468 |
+
pre-move focal cell), isolating the agent's own contribution from the
|
| 469 |
+
predator's chase. Positive = moved away, negative = moved toward.
|
| 470 |
+
"""
|
| 471 |
+
del action # reward derives from the post-move focal cell, not the action string
|
| 472 |
+
if game.eliminated:
|
| 473 |
+
return _REWARD_CAPTURED
|
| 474 |
+
if game.survived:
|
| 475 |
+
return _REWARD_SURVIVED
|
| 476 |
+
if blocked:
|
| 477 |
+
return _REWARD_WALL_HIT
|
| 478 |
+
focal = game.focal_sprite
|
| 479 |
+
if focal is None:
|
| 480 |
+
return 0.0
|
| 481 |
+
focal_after = (focal.x, focal.y)
|
| 482 |
+
d_before = self._bfs_distance(game, focal_before, predator_before)
|
| 483 |
+
d_after = self._bfs_distance(game, focal_after, predator_before)
|
| 484 |
+
if d_before is None or d_after is None:
|
| 485 |
+
return 0.0
|
| 486 |
+
return float(d_after - d_before)
|
| 487 |
+
|
| 488 |
+
def safety_distance(self, game: MotiveGridGame) -> int | None:
|
| 489 |
+
"""Return the BFS distance from the focal to the predator (None if N/A)."""
|
| 490 |
+
focal = game.focal_sprite
|
| 491 |
+
predator = game.predator_sprite
|
| 492 |
+
if focal is None or predator is None:
|
| 493 |
+
return None
|
| 494 |
+
return self._bfs_distance(game, (focal.x, focal.y), (predator.x, predator.y))
|
| 495 |
+
|
| 496 |
# ------------------------------------------------------------------ #
|
| 497 |
# ASCII legend
|
| 498 |
# ------------------------------------------------------------------ #
|
proteus/runtime/session.py
CHANGED
|
@@ -8,9 +8,10 @@ One session:
|
|
| 8 |
observation -> (optional probe) -> act -> apply -> record.
|
| 9 |
4. Stop on elimination / survival / budget; emit a SessionTrace.
|
| 10 |
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
|
|
|
| 14 |
"""
|
| 15 |
|
| 16 |
from __future__ import annotations
|
|
@@ -32,13 +33,6 @@ _PROBE_QUESTION = (
|
|
| 32 |
"Where is the predator, and which direction should you move to stay safe?"
|
| 33 |
)
|
| 34 |
|
| 35 |
-
# Reward schedule (mirrors the legacy MotiveGridModule).
|
| 36 |
-
_REWARD_CAPTURED = -50.0
|
| 37 |
-
_REWARD_SURVIVED = 50.0
|
| 38 |
-
_REWARD_WALL_HIT = -3.0
|
| 39 |
-
_REWARD_OPTIMAL = 5.0
|
| 40 |
-
_REWARD_SUBOPTIMAL = -2.0
|
| 41 |
-
|
| 42 |
_HANDOVER_FRAMING = (
|
| 43 |
"\n\nYou have been watching the game so far. From now on YOU control the "
|
| 44 |
"focal agent. Continue playing to survive."
|
|
@@ -116,7 +110,10 @@ class SessionRunner:
|
|
| 116 |
result = self._agent.act(observation, list(_ACTIONS), system_prompt)
|
| 117 |
|
| 118 |
blocked = self._apply(result.action)
|
| 119 |
-
reward = self.
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
turns.append(
|
| 122 |
TurnTrace(
|
|
@@ -230,18 +227,6 @@ class SessionRunner:
|
|
| 230 |
post = (moved.x, moved.y) if moved else None
|
| 231 |
return action in {"up", "down", "left", "right"} and post == pre
|
| 232 |
|
| 233 |
-
def _reward(self, action: str, optimal: str, blocked: bool) -> float:
|
| 234 |
-
assert self._game is not None
|
| 235 |
-
if self._game.eliminated:
|
| 236 |
-
return _REWARD_CAPTURED
|
| 237 |
-
if self._game.survived:
|
| 238 |
-
return _REWARD_SURVIVED
|
| 239 |
-
if blocked:
|
| 240 |
-
return _REWARD_WALL_HIT
|
| 241 |
-
if action == optimal:
|
| 242 |
-
return _REWARD_OPTIMAL
|
| 243 |
-
return _REWARD_SUBOPTIMAL
|
| 244 |
-
|
| 245 |
def _render(self) -> str:
|
| 246 |
assert self._scenario is not None and self._game is not None
|
| 247 |
return frame_to_ascii(self._game.current_grid(), self._scenario.legend())
|
|
|
|
| 8 |
observation -> (optional probe) -> act -> apply -> record.
|
| 9 |
4. Stop on elimination / survival / budget; emit a SessionTrace.
|
| 10 |
|
| 11 |
+
Per-turn reward is delegated to ``Scenario.step_reward`` (category-specific —
|
| 12 |
+
e.g. survival rewards moving away from the predator); the play budget mirrors
|
| 13 |
+
the legacy MotiveGridModule so the ported physics keep their validated meaning,
|
| 14 |
+
and all forfeit/risk machinery is gone.
|
| 15 |
"""
|
| 16 |
|
| 17 |
from __future__ import annotations
|
|
|
|
| 33 |
"Where is the predator, and which direction should you move to stay safe?"
|
| 34 |
)
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
_HANDOVER_FRAMING = (
|
| 37 |
"\n\nYou have been watching the game so far. From now on YOU control the "
|
| 38 |
"focal agent. Continue playing to survive."
|
|
|
|
| 110 |
result = self._agent.act(observation, list(_ACTIONS), system_prompt)
|
| 111 |
|
| 112 |
blocked = self._apply(result.action)
|
| 113 |
+
reward = self._scenario.step_reward(
|
| 114 |
+
self._game, result.action, blocked,
|
| 115 |
+
focal_before=focal_pos, predator_before=predator_pos,
|
| 116 |
+
)
|
| 117 |
|
| 118 |
turns.append(
|
| 119 |
TurnTrace(
|
|
|
|
| 227 |
post = (moved.x, moved.y) if moved else None
|
| 228 |
return action in {"up", "down", "left", "right"} and post == pre
|
| 229 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
def _render(self) -> str:
|
| 231 |
assert self._scenario is not None and self._game is not None
|
| 232 |
return frame_to_ascii(self._game.current_grid(), self._scenario.legend())
|
tests/grid/test_step_reward.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import random
|
| 2 |
+
|
| 3 |
+
from proteus.grid.difficulty import Difficulty
|
| 4 |
+
from proteus.grid.game import MotiveGridGame
|
| 5 |
+
from proteus.grid.scenario import get_scenario
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _handover():
|
| 9 |
+
s = get_scenario("predator_evade")()
|
| 10 |
+
game = MotiveGridGame(s, random.Random(42), Difficulty.EASY, max_steps=20)
|
| 11 |
+
for _ in range(s.cut_length(Difficulty.EASY)):
|
| 12 |
+
a = s.cut_focal_policy(game)
|
| 13 |
+
game.apply_motive_action(a)
|
| 14 |
+
s.record_focal_move(a)
|
| 15 |
+
return game, s
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def test_step_reward_positive_when_moving_away():
|
| 19 |
+
game, s = _handover()
|
| 20 |
+
focal_before = (game.focal_sprite.x, game.focal_sprite.y)
|
| 21 |
+
predator_before = (game.predator_sprite.x, game.predator_sprite.y)
|
| 22 |
+
# 'up' is the optimal escape at the EASY handover -> moves away.
|
| 23 |
+
game.apply_motive_action("up")
|
| 24 |
+
r = s.step_reward(game, "up", blocked=False,
|
| 25 |
+
focal_before=focal_before, predator_before=predator_before)
|
| 26 |
+
assert r > 0
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def test_step_reward_negative_when_moving_toward():
|
| 30 |
+
game, s = _handover()
|
| 31 |
+
focal_before = (game.focal_sprite.x, game.focal_sprite.y)
|
| 32 |
+
predator_before = (game.predator_sprite.x, game.predator_sprite.y)
|
| 33 |
+
# 'right' moves toward the predator (east).
|
| 34 |
+
game.apply_motive_action("right")
|
| 35 |
+
r = s.step_reward(game, "right", blocked=False,
|
| 36 |
+
focal_before=focal_before, predator_before=predator_before)
|
| 37 |
+
assert r < 0
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def test_step_reward_negative_on_wall_hit():
|
| 41 |
+
game, s = _handover()
|
| 42 |
+
focal_before = (game.focal_sprite.x, game.focal_sprite.y)
|
| 43 |
+
predator_before = (game.predator_sprite.x, game.predator_sprite.y)
|
| 44 |
+
# 'left' is blocked by the dead-end wall at the EASY handover.
|
| 45 |
+
game.apply_motive_action("left")
|
| 46 |
+
# The dead-end wall makes 'left' a no-op; confirm the engine blocked it
|
| 47 |
+
# so blocked=True faithfully reflects what _apply would have computed.
|
| 48 |
+
assert (game.focal_sprite.x, game.focal_sprite.y) == focal_before
|
| 49 |
+
r = s.step_reward(game, "left", blocked=True,
|
| 50 |
+
focal_before=focal_before, predator_before=predator_before)
|
| 51 |
+
assert r < 0
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def test_safety_distance_is_bfs_focal_to_predator():
|
| 55 |
+
game, s = _handover()
|
| 56 |
+
d = s.safety_distance(game)
|
| 57 |
+
assert isinstance(d, int) and d >= 0
|