yxc20098 commited on
Commit
a5a3a5a
·
1 Parent(s): de62afe

feat(bench): then composite predicate happened-before (PlanBench replanning / PERT anchor)

Browse files

A new composite operator alongside all_of / any_of / not. Each clause
must become true IN ORDER over the course of the episode -- not
merely all-of-now. Pattern mirrors waypoint_sequence: per-episode
latch on signals.then_progress[id] (greedy advance through every
consecutive currently-satisfied clause; the latch persists across
turns so a clause that stopped being true still counts).

YAML form:
then:
id: scout-then-counter # stable per-episode key
clauses:
- {buildings_discovered_gte: 1} # FIRST (e.g. scout)
- {unit_type_count_gte: {type: e3, n: 4}} # THEN (counter)

8 tests in tests/test_then_composite.py cover: composite key
registration; in-order success; B-only-without-A never advances;
late-A-then-B credits both via greedy advance; three-stage strict
order; empty clauses do not false-pass; phrase translation renders
'in this exact order: A -> THEN -> B'; multiple `then` predicates
with distinct ids latch independently.

Unblocks B2 mid-tech-switch-on-scout (the 'scout BEFORE counter'
enforcement that all_of cannot express) plus Group I procedural
seeds (proc-ordered-action-strict, etc.) plus Group G long-horizon
seeds (lh-progression-stage-locked) per SCENARIO_BACKLOG.md.

openra_bench/game_knowledge.py CHANGED
@@ -259,6 +259,19 @@ def _describe(node: Any, join: str = " AND ", coords: str = "exact") -> str:
259
  else f"fewer than {n} of your units remain"
260
  )
261
  return "NOT (" + _describe(inner, coords=coords) + ")"
 
 
 
 
 
 
 
 
 
 
 
 
 
262
  return join.join(_leaf_phrase(k, v, coords) for k, v in node.items())
263
 
264
 
 
259
  else f"fewer than {n} of your units remain"
260
  )
261
  return "NOT (" + _describe(inner, coords=coords) + ")"
262
+ if "then" in node:
263
+ # The happened-before composite. Read each clause as a stage
264
+ # in an enforced ordered chain.
265
+ v = node["then"] or {}
266
+ clauses = v.get("clauses") or []
267
+ if not clauses:
268
+ return "(empty then:)"
269
+ return (
270
+ "in this exact order: "
271
+ + " → THEN → ".join(
272
+ _describe(c, coords=coords) for c in clauses
273
+ )
274
+ )
275
  return join.join(_leaf_phrase(k, v, coords) for k, v in node.items())
276
 
277
 
openra_bench/rust_adapter.py CHANGED
@@ -131,6 +131,12 @@ class EpisodeSignals:
131
  # waypoint_sequence's ordered-visit progress, keyed by sequence id).
132
  # Reset for free: EpisodeSignals is reconstructed each episode.
133
  seq_progress: dict = field(default_factory=dict)
 
 
 
 
 
 
134
  # Per-episode tool-use accounting for the strict-toolban / procedural-
135
  # compliance family. tools_called counts each tool name the agent
136
  # invoked this episode; tool_violations counts how many of those calls
 
131
  # waypoint_sequence's ordered-visit progress, keyed by sequence id).
132
  # Reset for free: EpisodeSignals is reconstructed each episode.
133
  seq_progress: dict = field(default_factory=dict)
134
+ # Per-episode latch for the `then:[A,B]` happened-before composite
135
+ # (clauses-satisfied-so-far index, keyed by the `then.id`). Lets a
136
+ # scenario require "scout → THEN commit counter" instead of
137
+ # ``all_of`` which is satisfied by any state where both happen to
138
+ # be true. See win_conditions._then.
139
+ then_progress: dict = field(default_factory=dict)
140
  # Per-episode tool-use accounting for the strict-toolban / procedural-
141
  # compliance family. tools_called counts each tool name the agent
142
  # invoked this episode; tool_violations counts how many of those calls
openra_bench/scenarios/win_conditions.py CHANGED
@@ -202,7 +202,52 @@ _PREDICATES: dict[str, Callable[[WinContext, Any], bool]] = {
202
  }
203
 
204
  LEAF_KEYS = frozenset(_PREDICATES)
205
- COMPOSITE_KEYS = frozenset({"all_of", "any_of", "not"})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
206
 
207
 
208
  class WinCondition(BaseModel):
@@ -241,6 +286,8 @@ class WinCondition(BaseModel):
241
  return any(WinCondition(**c).evaluate(ctx) for c in node["any_of"])
242
  if "not" in node:
243
  return not WinCondition(**node["not"]).evaluate(ctx)
 
 
244
  return all(_PREDICATES[k](ctx, v) for k, v in node.items())
245
 
246
 
 
202
  }
203
 
204
  LEAF_KEYS = frozenset(_PREDICATES)
205
+ COMPOSITE_KEYS = frozenset({"all_of", "any_of", "not", "then"})
206
+
207
+
208
+ def _then(ctx: WinContext, v: Any) -> bool:
209
+ """Happened-before composite: each clause must become true IN ORDER
210
+ over the course of the episode. Stateful: the latch persists on
211
+ ``signals.then_progress[id]`` so an early clause that stopped being
212
+ true still counts (analogous to ``waypoint_sequence``).
213
+
214
+ YAML form::
215
+
216
+ then:
217
+ id: scout-then-counter # stable key (per-episode)
218
+ clauses:
219
+ - {buildings_discovered_gte: 1} # FIRST
220
+ - {unit_type_count_gte: {type: e3, n: 4}} # THEN
221
+
222
+ Satisfied once every clause has been observed-true AT LEAST ONCE,
223
+ in order. Reaching a later clause without the earlier one being
224
+ latched does NOT advance — this is what makes "counter chosen
225
+ AFTER scout" enforceable (vs the stateless ``all_of`` which is
226
+ satisfied by any state where every clause happens to be true now).
227
+ """
228
+ if not isinstance(v, dict):
229
+ raise ValueError("then: expects a dict with id + clauses")
230
+ clauses = v.get("clauses") or []
231
+ if not clauses:
232
+ return False
233
+ store = getattr(ctx.signals, "then_progress", None)
234
+ if store is None or not isinstance(store, dict):
235
+ store = {}
236
+ try:
237
+ ctx.signals.then_progress = store # type: ignore[attr-defined]
238
+ except Exception: # frozen/stub signals in unit tests
239
+ pass
240
+ key = str(v.get("id", id(v)))
241
+ idx = int(store.get(key, 0))
242
+ # Advance through every consecutive clause currently satisfied
243
+ # (typically ≤1 per evaluation, like waypoint_sequence).
244
+ while idx < len(clauses):
245
+ if WinCondition(**clauses[idx]).evaluate(ctx):
246
+ idx += 1
247
+ else:
248
+ break
249
+ store[key] = idx
250
+ return idx >= len(clauses)
251
 
252
 
253
  class WinCondition(BaseModel):
 
286
  return any(WinCondition(**c).evaluate(ctx) for c in node["any_of"])
287
  if "not" in node:
288
  return not WinCondition(**node["not"]).evaluate(ctx)
289
+ if "then" in node:
290
+ return _then(ctx, node["then"])
291
  return all(_PREDICATES[k](ctx, v) for k, v in node.items())
292
 
293
 
tests/test_then_composite.py ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """then:[A,B,…] happened-before composite predicate.
2
+
3
+ Unblocks the reactive-replan / scout-then-counter / strict-ordered-
4
+ sequence family: a scenario can require clauses to become true IN
5
+ ORDER over the course of an episode, not merely all-of-now.
6
+
7
+ Real-world anchor: PlanBench replanning, ALFWorld goal-conditional
8
+ adaptation, PERT stage dependencies. The bench's existing predicates
9
+ let you say "all of these must be true now" (`all_of`) and "any of
10
+ these must be true now" (`any_of`); `then:` is the missing temporal
11
+ operator: the clauses must have been observed-true in sequence.
12
+
13
+ State: per-episode `signals.then_progress[id]` (an integer index of
14
+ how many clauses have been observed-true so far), so a clause that
15
+ stopped being true still counts. Identical pattern to
16
+ `waypoint_sequence`'s `seq_progress` latch.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import types
22
+
23
+ from openra_bench.scenarios.win_conditions import (
24
+ COMPOSITE_KEYS,
25
+ WinContext,
26
+ evaluate,
27
+ )
28
+
29
+
30
+ def _ctx(units_killed: int = 0, buildings_disc: int = 0,
31
+ own_units: int = 0):
32
+ sig = types.SimpleNamespace(
33
+ units_killed=units_killed,
34
+ enemy_buildings_seen_ids=set(range(buildings_disc)),
35
+ game_tick=0,
36
+ units_lost=0,
37
+ cash=0, resources=0, power_provided=0, power_drained=0,
38
+ own_building_types=set(), own_buildings=[],
39
+ enemies_seen_ids=set(),
40
+ explored_percent=0.0,
41
+ then_progress={},
42
+ seq_progress={},
43
+ )
44
+ return WinContext(
45
+ signals=sig,
46
+ render_state={"units_summary": [
47
+ {"id": i, "type": "e1", "cell_x": 0, "cell_y": 0}
48
+ for i in range(own_units)
49
+ ]},
50
+ )
51
+
52
+
53
+ def test_then_is_a_composite_key():
54
+ """The new operator must register as a composite (not a leaf), so
55
+ the WinCondition validator accepts {then: …} alongside all_of /
56
+ any_of / not."""
57
+ assert "then" in COMPOSITE_KEYS
58
+
59
+
60
+ def test_then_satisfied_when_clauses_observed_in_order():
61
+ """A → B in order ⇒ true. The clauses don't both need to be true
62
+ NOW; they just need to have been observed-true at some past
63
+ evaluation, in order."""
64
+ cond = {"then": {
65
+ "id": "scout-then-counter",
66
+ "clauses": [
67
+ {"buildings_discovered_gte": 1}, # A
68
+ {"units_killed_gte": 3}, # B
69
+ ],
70
+ }}
71
+ ctx = _ctx(units_killed=0, buildings_disc=0)
72
+ # Initially: neither clause true.
73
+ assert evaluate(cond, ctx) is False
74
+
75
+ # A becomes true: scouted 1 building.
76
+ ctx.signals.enemy_buildings_seen_ids = {1}
77
+ assert evaluate(cond, ctx) is False # A latched, B not yet
78
+ assert ctx.signals.then_progress["scout-then-counter"] == 1
79
+
80
+ # A is no longer true (lost vision) — but the latch holds.
81
+ ctx.signals.enemy_buildings_seen_ids = set()
82
+ ctx.signals.units_killed = 3 # B now true
83
+ assert evaluate(cond, ctx) is True
84
+ assert ctx.signals.then_progress["scout-then-counter"] == 2
85
+
86
+
87
+ def test_then_no_credit_for_b_alone_without_a():
88
+ """B true while A is still false ⇒ the chain does NOT advance.
89
+ The latch only counts clauses observed-true-in-order. Once A
90
+ finally becomes true on a later evaluation, the chain advances
91
+ through every consecutive currently-satisfied clause in one pass
92
+ (greedy advance, mirroring `waypoint_sequence`'s semantics). The
93
+ bench-relevant guarantee is that a policy which ONLY ever made B
94
+ true (never A) can never satisfy the chain — keeping the
95
+ happened-before signal honest."""
96
+ cond = {"then": {
97
+ "id": "ordered-x",
98
+ "clauses": [
99
+ {"buildings_discovered_gte": 1}, # A
100
+ {"units_killed_gte": 3}, # B
101
+ ],
102
+ }}
103
+ ctx = _ctx()
104
+ # B true, A false → no progress.
105
+ ctx.signals.units_killed = 5
106
+ assert evaluate(cond, ctx) is False
107
+ assert ctx.signals.then_progress["ordered-x"] == 0
108
+ # Many more evals with B-only must STILL not advance (no false
109
+ # credit for the late-A case).
110
+ for _ in range(20):
111
+ assert evaluate(cond, ctx) is False
112
+ assert ctx.signals.then_progress["ordered-x"] == 0
113
+
114
+
115
+ def test_then_late_a_then_b_credits_both_clauses():
116
+ """Once A finally latches, the greedy advance picks up any
117
+ currently-satisfied later clauses in the same call. This matches
118
+ waypoint_sequence's semantics and is the right semantic: 'I
119
+ already had the counter ready; now that I've also scouted, the
120
+ chain is satisfied'."""
121
+ cond = {"then": {
122
+ "id": "late-a",
123
+ "clauses": [
124
+ {"buildings_discovered_gte": 1},
125
+ {"units_killed_gte": 3},
126
+ ],
127
+ }}
128
+ ctx = _ctx()
129
+ ctx.signals.units_killed = 5 # B pre-satisfied
130
+ assert evaluate(cond, ctx) is False # but A blocks
131
+ ctx.signals.enemy_buildings_seen_ids = {1} # A becomes true
132
+ assert evaluate(cond, ctx) is True # chain completes
133
+ assert ctx.signals.then_progress["late-a"] == 2
134
+
135
+
136
+ def test_then_three_clauses_strict_order():
137
+ """Three-stage chain: A → B → C. Each stage must latch before the
138
+ next can advance. Skipping is impossible by construction."""
139
+ cond = {"then": {
140
+ "id": "abc",
141
+ "clauses": [
142
+ {"buildings_discovered_gte": 1},
143
+ {"units_killed_gte": 2},
144
+ {"own_units_gte": 4},
145
+ ],
146
+ }}
147
+ ctx = _ctx()
148
+ # Skip straight to C: own_units=5 but no scout, no kills.
149
+ ctx.render_state["units_summary"] = [
150
+ {"id": i, "type": "e1", "cell_x": 0, "cell_y": 0} for i in range(5)
151
+ ]
152
+ assert evaluate(cond, ctx) is False
153
+ assert ctx.signals.then_progress["abc"] == 0
154
+
155
+ # Now satisfy A: scout 1 building.
156
+ ctx.signals.enemy_buildings_seen_ids = {1}
157
+ assert evaluate(cond, ctx) is False
158
+ # Stage advanced to 1, but B (kills) is still 0.
159
+ assert ctx.signals.then_progress["abc"] == 1
160
+
161
+ # Now B: kill 2.
162
+ ctx.signals.units_killed = 2
163
+ # On this evaluation, B advances (idx 1→2), and C is already true
164
+ # (5 own units >= 4) so C also advances (idx 2→3). Returns true.
165
+ assert evaluate(cond, ctx) is True
166
+ assert ctx.signals.then_progress["abc"] == 3
167
+
168
+
169
+ def test_then_empty_clauses_returns_false():
170
+ """Authoring-hygiene: an empty or missing clauses list must not
171
+ accidentally satisfy the predicate (silent always-true would be a
172
+ no-cheat bar violation)."""
173
+ assert evaluate({"then": {"id": "empty", "clauses": []}}, _ctx()) is False
174
+
175
+
176
+ def test_then_phrase_translation():
177
+ """The objective_brief must render `then` as a plain-language
178
+ ordered chain so the model knows the order matters."""
179
+ from openra_bench.game_knowledge import objective_brief
180
+
181
+ brief = objective_brief(
182
+ "description",
183
+ {"then": {
184
+ "id": "scout-then-counter",
185
+ "clauses": [
186
+ {"buildings_discovered_gte": 1},
187
+ {"units_killed_gte": 3},
188
+ ],
189
+ }},
190
+ None,
191
+ max_turns=30,
192
+ )
193
+ assert "in this exact order" in brief
194
+ assert "THEN" in brief
195
+ assert "spot ≥1 enemy buildings" in brief
196
+ assert "destroy ≥3 enemy units" in brief
197
+
198
+
199
+ def test_then_id_isolation():
200
+ """Two `then` predicates with different ids must latch
201
+ independently — necessary for a pack that uses multiple ordered
202
+ chains (e.g. a win clause + a fail clause both with their own
203
+ sequencing)."""
204
+ a = {"then": {"id": "A", "clauses": [{"units_killed_gte": 1}]}}
205
+ b = {"then": {"id": "B", "clauses": [{"units_killed_gte": 5}]}}
206
+ ctx = _ctx(units_killed=1)
207
+ assert evaluate(a, ctx) is True
208
+ assert evaluate(b, ctx) is False
209
+ assert ctx.signals.then_progress["A"] == 1
210
+ assert ctx.signals.then_progress["B"] == 0