irregular6612 Claude Sonnet 4.6 commited on
Commit
cc2f936
·
1 Parent(s): 14842f9

feat(cp6): trajectory + category-reward metrics (away_move_fraction, trajectory_agreement, ...)

Browse files

Add four new metric keys to compute_metrics: away_move_fraction (headline
survival eval — % turns with positive step_reward), mean_step_reward,
trajectory_agreement (% turns matching optimal-rollout focal position), and
final_distance_gap (|realized − optimal| BFS distance). Wire optimal_rollout
into SessionRunner so trajectory metrics are populated automatically. Retain all
four existing metrics unchanged. Extend _turn test helper with optional focal_pos
param (backward-compatible); rename and update empty-session test to cover all 8
keys.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

proteus/runtime/metrics.py CHANGED
@@ -10,6 +10,14 @@ Metrics (all in percent unless noted):
10
  - first_divergence_turn: the 1-based index of the first turn whose action
11
  diverged from the motive_action (a coarse trajectory-divergence proxy);
12
  0.0 if the agent never diverged (or there were no turns).
 
 
 
 
 
 
 
 
13
  """
14
 
15
  from __future__ import annotations
@@ -23,6 +31,9 @@ def compute_metrics(
23
  played_turns: int,
24
  play_turns: int,
25
  outcome: str,
 
 
 
26
  ) -> dict[str, float]:
27
  """Return the session metric dict.
28
 
@@ -32,6 +43,14 @@ def compute_metrics(
32
  play_turns: The survival budget (turns to survive for a win).
33
  outcome: ``"survived"`` or ``"eliminated"`` (reserved for future
34
  outcome-weighted metrics; unused in the base computation).
 
 
 
 
 
 
 
 
35
  """
36
  del outcome # reserved; base metrics are outcome-independent
37
 
@@ -42,6 +61,10 @@ def compute_metrics(
42
  "reactivity_index": 0.0,
43
  "survival_fraction": 0.0,
44
  "first_divergence_turn": 0.0,
 
 
 
 
45
  }
46
 
47
  congruent = sum(1 for t in turns if t.was_congruent)
@@ -58,9 +81,34 @@ def compute_metrics(
58
  if play_turns > 0:
59
  survival = min(played_turns / play_turns * 100.0, 100.0)
60
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  return {
62
  "motive_reading_accuracy": congruent / n * 100.0,
63
  "reactivity_index": (diag_congruent / len(diagnostic) * 100.0) if diagnostic else 0.0,
64
  "survival_fraction": survival,
65
  "first_divergence_turn": float(first_divergence),
 
 
 
 
66
  }
 
10
  - first_divergence_turn: the 1-based index of the first turn whose action
11
  diverged from the motive_action (a coarse trajectory-divergence proxy);
12
  0.0 if the agent never diverged (or there were no turns).
13
+ - away_move_fraction: % of played turns whose step_reward was positive
14
+ (agent moved away from the predator) — headline survival eval.
15
+ - mean_step_reward: mean per-turn reward across all played turns.
16
+ - trajectory_agreement: % of turns whose realized focal position equals
17
+ the optimal-rollout focal position at the same step; 0.0 when not
18
+ provided (None or empty optimal_focal_positions).
19
+ - final_distance_gap: |realized_final_safety − optimal_final_safety| in
20
+ BFS distance units; 0.0 when rollout safety data are not supplied.
21
  """
22
 
23
  from __future__ import annotations
 
31
  played_turns: int,
32
  play_turns: int,
33
  outcome: str,
34
+ optimal_focal_positions: list[tuple[int, int]] | None = None,
35
+ realized_final_safety: int | None = None,
36
+ optimal_final_safety: int | None = None,
37
  ) -> dict[str, float]:
38
  """Return the session metric dict.
39
 
 
43
  play_turns: The survival budget (turns to survive for a win).
44
  outcome: ``"survived"`` or ``"eliminated"`` (reserved for future
45
  outcome-weighted metrics; unused in the base computation).
46
+ optimal_focal_positions: Pre-move focal positions from the optimal
47
+ rollout, aligned index-for-index with ``turns``. When not
48
+ provided (None or empty), ``trajectory_agreement`` is 0.0.
49
+ realized_final_safety: BFS safety distance of the realized session
50
+ at end-of-play. Required (with ``optimal_final_safety``) to
51
+ compute ``final_distance_gap``; otherwise 0.0.
52
+ optimal_final_safety: BFS safety distance of the optimal rollout at
53
+ end-of-play.
54
  """
55
  del outcome # reserved; base metrics are outcome-independent
56
 
 
61
  "reactivity_index": 0.0,
62
  "survival_fraction": 0.0,
63
  "first_divergence_turn": 0.0,
64
+ "away_move_fraction": 0.0,
65
+ "mean_step_reward": 0.0,
66
+ "trajectory_agreement": 0.0,
67
+ "final_distance_gap": 0.0,
68
  }
69
 
70
  congruent = sum(1 for t in turns if t.was_congruent)
 
81
  if play_turns > 0:
82
  survival = min(played_turns / play_turns * 100.0, 100.0)
83
 
84
+ # reward > 0 counts as "moved away"; the terminal survive turn (+50 bonus)
85
+ # also counts, so away_move_fraction can exceed the share of true away-moves.
86
+ away_moves = sum(1 for t in turns if t.reward > 0)
87
+ away_move_fraction = away_moves / n * 100.0
88
+ mean_step_reward = sum(t.reward for t in turns) / n
89
+
90
+ trajectory_agreement = 0.0
91
+ if optimal_focal_positions:
92
+ realized = [tuple(t.focal_pos) for t in turns]
93
+ optimal = [tuple(p) for p in optimal_focal_positions]
94
+ compared = min(len(realized), len(optimal))
95
+ agree = sum(1 for i in range(compared) if realized[i] == optimal[i])
96
+ # Divide by n (all played turns), not `compared`: turns beyond the
97
+ # optimal rollout's length have no counterpart and count as misses
98
+ # (a longer-than-optimal realized path is penalised). Spec §5.4.
99
+ trajectory_agreement = agree / n * 100.0
100
+
101
+ final_distance_gap = 0.0
102
+ if realized_final_safety is not None and optimal_final_safety is not None:
103
+ final_distance_gap = float(abs(realized_final_safety - optimal_final_safety))
104
+
105
  return {
106
  "motive_reading_accuracy": congruent / n * 100.0,
107
  "reactivity_index": (diag_congruent / len(diagnostic) * 100.0) if diagnostic else 0.0,
108
  "survival_fraction": survival,
109
  "first_divergence_turn": float(first_divergence),
110
+ "away_move_fraction": away_move_fraction,
111
+ "mean_step_reward": mean_step_reward,
112
+ "trajectory_agreement": trajectory_agreement,
113
+ "final_distance_gap": final_distance_gap,
114
  }
proteus/runtime/session.py CHANGED
@@ -24,6 +24,7 @@ from proteus.grid.difficulty import Difficulty
24
  from proteus.grid.game import MotiveGridGame
25
  from proteus.grid.scenario import Scenario, get_scenario
26
  from proteus.runtime.metrics import compute_metrics
 
27
  from proteus.runtime.trace import SessionTrace, TurnTrace
28
 
29
  _ACTIONS = ["up", "down", "left", "right", "stay"]
@@ -152,11 +153,18 @@ class SessionRunner:
152
  ), "play loop exited without a terminal state or exhausting the budget"
153
 
154
  outcome = "eliminated" if self._game.eliminated else "survived"
 
 
 
 
155
  metrics = compute_metrics(
156
  turns,
157
  played_turns=len(turns),
158
  play_turns=self._play_turns,
159
  outcome=outcome,
 
 
 
160
  )
161
  return SessionTrace(
162
  scenario=self._scenario_name,
 
24
  from proteus.grid.game import MotiveGridGame
25
  from proteus.grid.scenario import Scenario, get_scenario
26
  from proteus.runtime.metrics import compute_metrics
27
+ from proteus.runtime.rollout import optimal_rollout
28
  from proteus.runtime.trace import SessionTrace, TurnTrace
29
 
30
  _ACTIONS = ["up", "down", "left", "right", "stay"]
 
153
  ), "play loop exited without a terminal state or exhausting the budget"
154
 
155
  outcome = "eliminated" if self._game.eliminated else "survived"
156
+ rollout = optimal_rollout(
157
+ self._scenario_name, self._seed, self._difficulty, len(turns),
158
+ )
159
+ realized_final_safety = self._scenario.safety_distance(self._game)
160
  metrics = compute_metrics(
161
  turns,
162
  played_turns=len(turns),
163
  play_turns=self._play_turns,
164
  outcome=outcome,
165
+ optimal_focal_positions=rollout.focal_positions,
166
+ realized_final_safety=realized_final_safety,
167
+ optimal_final_safety=rollout.final_safety_distance,
168
  )
169
  return SessionTrace(
170
  scenario=self._scenario_name,
tests/runtime/test_metrics.py CHANGED
@@ -2,12 +2,12 @@ from proteus.runtime.trace import TurnTrace, SessionTrace
2
  from proteus.runtime.metrics import compute_metrics
3
 
4
 
5
- def _turn(idx, action, motive, habit, reward):
6
  return TurnTrace(
7
  turn_idx=idx, observation="", action=action, motive_action=motive,
8
  habit_action=habit, is_diagnostic=(motive != habit),
9
  was_congruent=(action == motive), reward=reward,
10
- focal_pos=(0, 0), predator_pos=(1, 1),
11
  )
12
 
13
 
@@ -67,11 +67,71 @@ def test_survival_fraction_capped_at_100():
67
  assert m["survival_fraction"] == 100.0
68
 
69
 
70
- def test_empty_session_all_four_keys_zero():
71
  m = compute_metrics([], played_turns=0, play_turns=10, outcome="eliminated")
72
  assert m == {
73
  "motive_reading_accuracy": 0.0,
74
  "reactivity_index": 0.0,
75
  "survival_fraction": 0.0,
76
  "first_divergence_turn": 0.0,
 
 
 
 
77
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  from proteus.runtime.metrics import compute_metrics
3
 
4
 
5
+ def _turn(idx, action, motive, habit, reward, focal_pos=(0, 0)):
6
  return TurnTrace(
7
  turn_idx=idx, observation="", action=action, motive_action=motive,
8
  habit_action=habit, is_diagnostic=(motive != habit),
9
  was_congruent=(action == motive), reward=reward,
10
+ focal_pos=focal_pos, predator_pos=(1, 1),
11
  )
12
 
13
 
 
67
  assert m["survival_fraction"] == 100.0
68
 
69
 
70
+ def test_empty_session_all_keys_zero():
71
  m = compute_metrics([], played_turns=0, play_turns=10, outcome="eliminated")
72
  assert m == {
73
  "motive_reading_accuracy": 0.0,
74
  "reactivity_index": 0.0,
75
  "survival_fraction": 0.0,
76
  "first_divergence_turn": 0.0,
77
+ "away_move_fraction": 0.0,
78
+ "mean_step_reward": 0.0,
79
+ "trajectory_agreement": 0.0,
80
+ "final_distance_gap": 0.0,
81
  }
82
+
83
+
84
+ def test_away_move_fraction_and_mean_reward():
85
+ turns = [
86
+ _turn(1, "up", "up", "left", 1.0, (3, 3)), # away (reward > 0)
87
+ _turn(2, "right", "up", "left", -1.0, (3, 2)), # toward (reward < 0)
88
+ _turn(3, "up", "up", "left", 2.0, (4, 2)), # away
89
+ ]
90
+ m = compute_metrics(turns, played_turns=3, play_turns=5, outcome="eliminated")
91
+ assert m["away_move_fraction"] == 2 / 3 * 100.0
92
+ assert m["mean_step_reward"] == (1.0 - 1.0 + 2.0) / 3
93
+
94
+
95
+ def test_trajectory_agreement_and_final_gap():
96
+ turns = [
97
+ _turn(1, "up", "up", "left", 1.0, (3, 3)),
98
+ _turn(2, "up", "up", "left", 1.0, (3, 2)),
99
+ ]
100
+ # Optimal pre-move positions: turn1 matches (3,3), turn2 differs (4,2) vs (3,2).
101
+ m = compute_metrics(
102
+ turns, played_turns=2, play_turns=5, outcome="eliminated",
103
+ optimal_focal_positions=[(3, 3), (4, 2)],
104
+ realized_final_safety=2, optimal_final_safety=4,
105
+ )
106
+ assert m["trajectory_agreement"] == 1 / 2 * 100.0
107
+ assert m["final_distance_gap"] == 2.0
108
+
109
+
110
+ def test_new_metric_keys_default_zero_without_rollout():
111
+ turns = [_turn(1, "up", "up", "left", 1.0, (3, 3))]
112
+ m = compute_metrics(turns, played_turns=1, play_turns=5, outcome="survived")
113
+ # Trajectory keys default to 0.0 when no rollout data is supplied.
114
+ assert m["trajectory_agreement"] == 0.0
115
+ assert m["final_distance_gap"] == 0.0
116
+ assert set(m) >= {
117
+ "motive_reading_accuracy", "reactivity_index", "survival_fraction",
118
+ "first_divergence_turn", "away_move_fraction", "mean_step_reward",
119
+ "trajectory_agreement", "final_distance_gap",
120
+ }
121
+
122
+
123
+ def test_trajectory_agreement_penalizes_turns_beyond_optimal():
124
+ # Realized ran longer than the optimal rollout: the extra turn has no
125
+ # optimal counterpart and counts as a disagreement (denominator = n).
126
+ turns = [
127
+ _turn(1, "up", "up", "left", 1.0, (3, 3)),
128
+ _turn(2, "up", "up", "left", 1.0, (3, 2)),
129
+ _turn(3, "up", "up", "left", 1.0, (3, 1)),
130
+ ]
131
+ m = compute_metrics(
132
+ turns, played_turns=3, play_turns=5, outcome="survived",
133
+ optimal_focal_positions=[(3, 3), (3, 2)], # only 2 optimal positions
134
+ )
135
+ # turns 1-2 match optimal -> 2 agree; turn 3 has no optimal counterpart.
136
+ # agreement = 2/3 (divided by n=3, NOT by compared=2).
137
+ assert m["trajectory_agreement"] == 2 / 3 * 100.0