Spaces:
Runtime error
Runtime error
Commit ·
cc2f936
1
Parent(s): 14842f9
feat(cp6): trajectory + category-reward metrics (away_move_fraction, trajectory_agreement, ...)
Browse filesAdd four new metric keys to compute_metrics: away_move_fraction (headline
survival eval — % turns with positive step_reward), mean_step_reward,
trajectory_agreement (% turns matching optimal-rollout focal position), and
final_distance_gap (|realized − optimal| BFS distance). Wire optimal_rollout
into SessionRunner so trajectory metrics are populated automatically. Retain all
four existing metrics unchanged. Extend _turn test helper with optional focal_pos
param (backward-compatible); rename and update empty-session test to cover all 8
keys.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
- proteus/runtime/metrics.py +48 -0
- proteus/runtime/session.py +8 -0
- tests/runtime/test_metrics.py +63 -3
proteus/runtime/metrics.py
CHANGED
|
@@ -10,6 +10,14 @@ Metrics (all in percent unless noted):
|
|
| 10 |
- first_divergence_turn: the 1-based index of the first turn whose action
|
| 11 |
diverged from the motive_action (a coarse trajectory-divergence proxy);
|
| 12 |
0.0 if the agent never diverged (or there were no turns).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
"""
|
| 14 |
|
| 15 |
from __future__ import annotations
|
|
@@ -23,6 +31,9 @@ def compute_metrics(
|
|
| 23 |
played_turns: int,
|
| 24 |
play_turns: int,
|
| 25 |
outcome: str,
|
|
|
|
|
|
|
|
|
|
| 26 |
) -> dict[str, float]:
|
| 27 |
"""Return the session metric dict.
|
| 28 |
|
|
@@ -32,6 +43,14 @@ def compute_metrics(
|
|
| 32 |
play_turns: The survival budget (turns to survive for a win).
|
| 33 |
outcome: ``"survived"`` or ``"eliminated"`` (reserved for future
|
| 34 |
outcome-weighted metrics; unused in the base computation).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
"""
|
| 36 |
del outcome # reserved; base metrics are outcome-independent
|
| 37 |
|
|
@@ -42,6 +61,10 @@ def compute_metrics(
|
|
| 42 |
"reactivity_index": 0.0,
|
| 43 |
"survival_fraction": 0.0,
|
| 44 |
"first_divergence_turn": 0.0,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
}
|
| 46 |
|
| 47 |
congruent = sum(1 for t in turns if t.was_congruent)
|
|
@@ -58,9 +81,34 @@ def compute_metrics(
|
|
| 58 |
if play_turns > 0:
|
| 59 |
survival = min(played_turns / play_turns * 100.0, 100.0)
|
| 60 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
return {
|
| 62 |
"motive_reading_accuracy": congruent / n * 100.0,
|
| 63 |
"reactivity_index": (diag_congruent / len(diagnostic) * 100.0) if diagnostic else 0.0,
|
| 64 |
"survival_fraction": survival,
|
| 65 |
"first_divergence_turn": float(first_divergence),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
}
|
|
|
|
| 10 |
- first_divergence_turn: the 1-based index of the first turn whose action
|
| 11 |
diverged from the motive_action (a coarse trajectory-divergence proxy);
|
| 12 |
0.0 if the agent never diverged (or there were no turns).
|
| 13 |
+
- away_move_fraction: % of played turns whose step_reward was positive
|
| 14 |
+
(agent moved away from the predator) — headline survival eval.
|
| 15 |
+
- mean_step_reward: mean per-turn reward across all played turns.
|
| 16 |
+
- trajectory_agreement: % of turns whose realized focal position equals
|
| 17 |
+
the optimal-rollout focal position at the same step; 0.0 when not
|
| 18 |
+
provided (None or empty optimal_focal_positions).
|
| 19 |
+
- final_distance_gap: |realized_final_safety − optimal_final_safety| in
|
| 20 |
+
BFS distance units; 0.0 when rollout safety data are not supplied.
|
| 21 |
"""
|
| 22 |
|
| 23 |
from __future__ import annotations
|
|
|
|
| 31 |
played_turns: int,
|
| 32 |
play_turns: int,
|
| 33 |
outcome: str,
|
| 34 |
+
optimal_focal_positions: list[tuple[int, int]] | None = None,
|
| 35 |
+
realized_final_safety: int | None = None,
|
| 36 |
+
optimal_final_safety: int | None = None,
|
| 37 |
) -> dict[str, float]:
|
| 38 |
"""Return the session metric dict.
|
| 39 |
|
|
|
|
| 43 |
play_turns: The survival budget (turns to survive for a win).
|
| 44 |
outcome: ``"survived"`` or ``"eliminated"`` (reserved for future
|
| 45 |
outcome-weighted metrics; unused in the base computation).
|
| 46 |
+
optimal_focal_positions: Pre-move focal positions from the optimal
|
| 47 |
+
rollout, aligned index-for-index with ``turns``. When not
|
| 48 |
+
provided (None or empty), ``trajectory_agreement`` is 0.0.
|
| 49 |
+
realized_final_safety: BFS safety distance of the realized session
|
| 50 |
+
at end-of-play. Required (with ``optimal_final_safety``) to
|
| 51 |
+
compute ``final_distance_gap``; otherwise 0.0.
|
| 52 |
+
optimal_final_safety: BFS safety distance of the optimal rollout at
|
| 53 |
+
end-of-play.
|
| 54 |
"""
|
| 55 |
del outcome # reserved; base metrics are outcome-independent
|
| 56 |
|
|
|
|
| 61 |
"reactivity_index": 0.0,
|
| 62 |
"survival_fraction": 0.0,
|
| 63 |
"first_divergence_turn": 0.0,
|
| 64 |
+
"away_move_fraction": 0.0,
|
| 65 |
+
"mean_step_reward": 0.0,
|
| 66 |
+
"trajectory_agreement": 0.0,
|
| 67 |
+
"final_distance_gap": 0.0,
|
| 68 |
}
|
| 69 |
|
| 70 |
congruent = sum(1 for t in turns if t.was_congruent)
|
|
|
|
| 81 |
if play_turns > 0:
|
| 82 |
survival = min(played_turns / play_turns * 100.0, 100.0)
|
| 83 |
|
| 84 |
+
# reward > 0 counts as "moved away"; the terminal survive turn (+50 bonus)
|
| 85 |
+
# also counts, so away_move_fraction can exceed the share of true away-moves.
|
| 86 |
+
away_moves = sum(1 for t in turns if t.reward > 0)
|
| 87 |
+
away_move_fraction = away_moves / n * 100.0
|
| 88 |
+
mean_step_reward = sum(t.reward for t in turns) / n
|
| 89 |
+
|
| 90 |
+
trajectory_agreement = 0.0
|
| 91 |
+
if optimal_focal_positions:
|
| 92 |
+
realized = [tuple(t.focal_pos) for t in turns]
|
| 93 |
+
optimal = [tuple(p) for p in optimal_focal_positions]
|
| 94 |
+
compared = min(len(realized), len(optimal))
|
| 95 |
+
agree = sum(1 for i in range(compared) if realized[i] == optimal[i])
|
| 96 |
+
# Divide by n (all played turns), not `compared`: turns beyond the
|
| 97 |
+
# optimal rollout's length have no counterpart and count as misses
|
| 98 |
+
# (a longer-than-optimal realized path is penalised). Spec §5.4.
|
| 99 |
+
trajectory_agreement = agree / n * 100.0
|
| 100 |
+
|
| 101 |
+
final_distance_gap = 0.0
|
| 102 |
+
if realized_final_safety is not None and optimal_final_safety is not None:
|
| 103 |
+
final_distance_gap = float(abs(realized_final_safety - optimal_final_safety))
|
| 104 |
+
|
| 105 |
return {
|
| 106 |
"motive_reading_accuracy": congruent / n * 100.0,
|
| 107 |
"reactivity_index": (diag_congruent / len(diagnostic) * 100.0) if diagnostic else 0.0,
|
| 108 |
"survival_fraction": survival,
|
| 109 |
"first_divergence_turn": float(first_divergence),
|
| 110 |
+
"away_move_fraction": away_move_fraction,
|
| 111 |
+
"mean_step_reward": mean_step_reward,
|
| 112 |
+
"trajectory_agreement": trajectory_agreement,
|
| 113 |
+
"final_distance_gap": final_distance_gap,
|
| 114 |
}
|
proteus/runtime/session.py
CHANGED
|
@@ -24,6 +24,7 @@ from proteus.grid.difficulty import Difficulty
|
|
| 24 |
from proteus.grid.game import MotiveGridGame
|
| 25 |
from proteus.grid.scenario import Scenario, get_scenario
|
| 26 |
from proteus.runtime.metrics import compute_metrics
|
|
|
|
| 27 |
from proteus.runtime.trace import SessionTrace, TurnTrace
|
| 28 |
|
| 29 |
_ACTIONS = ["up", "down", "left", "right", "stay"]
|
|
@@ -152,11 +153,18 @@ class SessionRunner:
|
|
| 152 |
), "play loop exited without a terminal state or exhausting the budget"
|
| 153 |
|
| 154 |
outcome = "eliminated" if self._game.eliminated else "survived"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 155 |
metrics = compute_metrics(
|
| 156 |
turns,
|
| 157 |
played_turns=len(turns),
|
| 158 |
play_turns=self._play_turns,
|
| 159 |
outcome=outcome,
|
|
|
|
|
|
|
|
|
|
| 160 |
)
|
| 161 |
return SessionTrace(
|
| 162 |
scenario=self._scenario_name,
|
|
|
|
| 24 |
from proteus.grid.game import MotiveGridGame
|
| 25 |
from proteus.grid.scenario import Scenario, get_scenario
|
| 26 |
from proteus.runtime.metrics import compute_metrics
|
| 27 |
+
from proteus.runtime.rollout import optimal_rollout
|
| 28 |
from proteus.runtime.trace import SessionTrace, TurnTrace
|
| 29 |
|
| 30 |
_ACTIONS = ["up", "down", "left", "right", "stay"]
|
|
|
|
| 153 |
), "play loop exited without a terminal state or exhausting the budget"
|
| 154 |
|
| 155 |
outcome = "eliminated" if self._game.eliminated else "survived"
|
| 156 |
+
rollout = optimal_rollout(
|
| 157 |
+
self._scenario_name, self._seed, self._difficulty, len(turns),
|
| 158 |
+
)
|
| 159 |
+
realized_final_safety = self._scenario.safety_distance(self._game)
|
| 160 |
metrics = compute_metrics(
|
| 161 |
turns,
|
| 162 |
played_turns=len(turns),
|
| 163 |
play_turns=self._play_turns,
|
| 164 |
outcome=outcome,
|
| 165 |
+
optimal_focal_positions=rollout.focal_positions,
|
| 166 |
+
realized_final_safety=realized_final_safety,
|
| 167 |
+
optimal_final_safety=rollout.final_safety_distance,
|
| 168 |
)
|
| 169 |
return SessionTrace(
|
| 170 |
scenario=self._scenario_name,
|
tests/runtime/test_metrics.py
CHANGED
|
@@ -2,12 +2,12 @@ from proteus.runtime.trace import TurnTrace, SessionTrace
|
|
| 2 |
from proteus.runtime.metrics import compute_metrics
|
| 3 |
|
| 4 |
|
| 5 |
-
def _turn(idx, action, motive, habit, reward):
|
| 6 |
return TurnTrace(
|
| 7 |
turn_idx=idx, observation="", action=action, motive_action=motive,
|
| 8 |
habit_action=habit, is_diagnostic=(motive != habit),
|
| 9 |
was_congruent=(action == motive), reward=reward,
|
| 10 |
-
focal_pos=
|
| 11 |
)
|
| 12 |
|
| 13 |
|
|
@@ -67,11 +67,71 @@ def test_survival_fraction_capped_at_100():
|
|
| 67 |
assert m["survival_fraction"] == 100.0
|
| 68 |
|
| 69 |
|
| 70 |
-
def
|
| 71 |
m = compute_metrics([], played_turns=0, play_turns=10, outcome="eliminated")
|
| 72 |
assert m == {
|
| 73 |
"motive_reading_accuracy": 0.0,
|
| 74 |
"reactivity_index": 0.0,
|
| 75 |
"survival_fraction": 0.0,
|
| 76 |
"first_divergence_turn": 0.0,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
from proteus.runtime.metrics import compute_metrics
|
| 3 |
|
| 4 |
|
| 5 |
+
def _turn(idx, action, motive, habit, reward, focal_pos=(0, 0)):
|
| 6 |
return TurnTrace(
|
| 7 |
turn_idx=idx, observation="", action=action, motive_action=motive,
|
| 8 |
habit_action=habit, is_diagnostic=(motive != habit),
|
| 9 |
was_congruent=(action == motive), reward=reward,
|
| 10 |
+
focal_pos=focal_pos, predator_pos=(1, 1),
|
| 11 |
)
|
| 12 |
|
| 13 |
|
|
|
|
| 67 |
assert m["survival_fraction"] == 100.0
|
| 68 |
|
| 69 |
|
| 70 |
+
def test_empty_session_all_keys_zero():
|
| 71 |
m = compute_metrics([], played_turns=0, play_turns=10, outcome="eliminated")
|
| 72 |
assert m == {
|
| 73 |
"motive_reading_accuracy": 0.0,
|
| 74 |
"reactivity_index": 0.0,
|
| 75 |
"survival_fraction": 0.0,
|
| 76 |
"first_divergence_turn": 0.0,
|
| 77 |
+
"away_move_fraction": 0.0,
|
| 78 |
+
"mean_step_reward": 0.0,
|
| 79 |
+
"trajectory_agreement": 0.0,
|
| 80 |
+
"final_distance_gap": 0.0,
|
| 81 |
}
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def test_away_move_fraction_and_mean_reward():
|
| 85 |
+
turns = [
|
| 86 |
+
_turn(1, "up", "up", "left", 1.0, (3, 3)), # away (reward > 0)
|
| 87 |
+
_turn(2, "right", "up", "left", -1.0, (3, 2)), # toward (reward < 0)
|
| 88 |
+
_turn(3, "up", "up", "left", 2.0, (4, 2)), # away
|
| 89 |
+
]
|
| 90 |
+
m = compute_metrics(turns, played_turns=3, play_turns=5, outcome="eliminated")
|
| 91 |
+
assert m["away_move_fraction"] == 2 / 3 * 100.0
|
| 92 |
+
assert m["mean_step_reward"] == (1.0 - 1.0 + 2.0) / 3
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def test_trajectory_agreement_and_final_gap():
|
| 96 |
+
turns = [
|
| 97 |
+
_turn(1, "up", "up", "left", 1.0, (3, 3)),
|
| 98 |
+
_turn(2, "up", "up", "left", 1.0, (3, 2)),
|
| 99 |
+
]
|
| 100 |
+
# Optimal pre-move positions: turn1 matches (3,3), turn2 differs (4,2) vs (3,2).
|
| 101 |
+
m = compute_metrics(
|
| 102 |
+
turns, played_turns=2, play_turns=5, outcome="eliminated",
|
| 103 |
+
optimal_focal_positions=[(3, 3), (4, 2)],
|
| 104 |
+
realized_final_safety=2, optimal_final_safety=4,
|
| 105 |
+
)
|
| 106 |
+
assert m["trajectory_agreement"] == 1 / 2 * 100.0
|
| 107 |
+
assert m["final_distance_gap"] == 2.0
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def test_new_metric_keys_default_zero_without_rollout():
|
| 111 |
+
turns = [_turn(1, "up", "up", "left", 1.0, (3, 3))]
|
| 112 |
+
m = compute_metrics(turns, played_turns=1, play_turns=5, outcome="survived")
|
| 113 |
+
# Trajectory keys default to 0.0 when no rollout data is supplied.
|
| 114 |
+
assert m["trajectory_agreement"] == 0.0
|
| 115 |
+
assert m["final_distance_gap"] == 0.0
|
| 116 |
+
assert set(m) >= {
|
| 117 |
+
"motive_reading_accuracy", "reactivity_index", "survival_fraction",
|
| 118 |
+
"first_divergence_turn", "away_move_fraction", "mean_step_reward",
|
| 119 |
+
"trajectory_agreement", "final_distance_gap",
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def test_trajectory_agreement_penalizes_turns_beyond_optimal():
|
| 124 |
+
# Realized ran longer than the optimal rollout: the extra turn has no
|
| 125 |
+
# optimal counterpart and counts as a disagreement (denominator = n).
|
| 126 |
+
turns = [
|
| 127 |
+
_turn(1, "up", "up", "left", 1.0, (3, 3)),
|
| 128 |
+
_turn(2, "up", "up", "left", 1.0, (3, 2)),
|
| 129 |
+
_turn(3, "up", "up", "left", 1.0, (3, 1)),
|
| 130 |
+
]
|
| 131 |
+
m = compute_metrics(
|
| 132 |
+
turns, played_turns=3, play_turns=5, outcome="survived",
|
| 133 |
+
optimal_focal_positions=[(3, 3), (3, 2)], # only 2 optimal positions
|
| 134 |
+
)
|
| 135 |
+
# turns 1-2 match optimal -> 2 agree; turn 3 has no optimal counterpart.
|
| 136 |
+
# agreement = 2/3 (divided by n=3, NOT by compared=2).
|
| 137 |
+
assert m["trajectory_agreement"] == 2 / 3 * 100.0
|