Spaces:
Running
feat(bench): then composite predicate happened-before (PlanBench replanning / PERT anchor)
Browse filesA new composite operator alongside all_of / any_of / not. Each clause
must become true IN ORDER over the course of the episode -- not
merely all-of-now. Pattern mirrors waypoint_sequence: per-episode
latch on signals.then_progress[id] (greedy advance through every
consecutive currently-satisfied clause; the latch persists across
turns so a clause that stopped being true still counts).
YAML form:
then:
id: scout-then-counter # stable per-episode key
clauses:
- {buildings_discovered_gte: 1} # FIRST (e.g. scout)
- {unit_type_count_gte: {type: e3, n: 4}} # THEN (counter)
8 tests in tests/test_then_composite.py cover: composite key
registration; in-order success; B-only-without-A never advances;
late-A-then-B credits both via greedy advance; three-stage strict
order; empty clauses do not false-pass; phrase translation renders
'in this exact order: A -> THEN -> B'; multiple `then` predicates
with distinct ids latch independently.
Unblocks B2 mid-tech-switch-on-scout (the 'scout BEFORE counter'
enforcement that all_of cannot express) plus Group I procedural
seeds (proc-ordered-action-strict, etc.) plus Group G long-horizon
seeds (lh-progression-stage-locked) per SCENARIO_BACKLOG.md.
|
@@ -259,6 +259,19 @@ def _describe(node: Any, join: str = " AND ", coords: str = "exact") -> str:
|
|
| 259 |
else f"fewer than {n} of your units remain"
|
| 260 |
)
|
| 261 |
return "NOT (" + _describe(inner, coords=coords) + ")"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 262 |
return join.join(_leaf_phrase(k, v, coords) for k, v in node.items())
|
| 263 |
|
| 264 |
|
|
|
|
| 259 |
else f"fewer than {n} of your units remain"
|
| 260 |
)
|
| 261 |
return "NOT (" + _describe(inner, coords=coords) + ")"
|
| 262 |
+
if "then" in node:
|
| 263 |
+
# The happened-before composite. Read each clause as a stage
|
| 264 |
+
# in an enforced ordered chain.
|
| 265 |
+
v = node["then"] or {}
|
| 266 |
+
clauses = v.get("clauses") or []
|
| 267 |
+
if not clauses:
|
| 268 |
+
return "(empty then:)"
|
| 269 |
+
return (
|
| 270 |
+
"in this exact order: "
|
| 271 |
+
+ " → THEN → ".join(
|
| 272 |
+
_describe(c, coords=coords) for c in clauses
|
| 273 |
+
)
|
| 274 |
+
)
|
| 275 |
return join.join(_leaf_phrase(k, v, coords) for k, v in node.items())
|
| 276 |
|
| 277 |
|
|
@@ -131,6 +131,12 @@ class EpisodeSignals:
|
|
| 131 |
# waypoint_sequence's ordered-visit progress, keyed by sequence id).
|
| 132 |
# Reset for free: EpisodeSignals is reconstructed each episode.
|
| 133 |
seq_progress: dict = field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
# Per-episode tool-use accounting for the strict-toolban / procedural-
|
| 135 |
# compliance family. tools_called counts each tool name the agent
|
| 136 |
# invoked this episode; tool_violations counts how many of those calls
|
|
|
|
| 131 |
# waypoint_sequence's ordered-visit progress, keyed by sequence id).
|
| 132 |
# Reset for free: EpisodeSignals is reconstructed each episode.
|
| 133 |
seq_progress: dict = field(default_factory=dict)
|
| 134 |
+
# Per-episode latch for the `then:[A,B]` happened-before composite
|
| 135 |
+
# (clauses-satisfied-so-far index, keyed by the `then.id`). Lets a
|
| 136 |
+
# scenario require "scout → THEN commit counter" instead of
|
| 137 |
+
# ``all_of`` which is satisfied by any state where both happen to
|
| 138 |
+
# be true. See win_conditions._then.
|
| 139 |
+
then_progress: dict = field(default_factory=dict)
|
| 140 |
# Per-episode tool-use accounting for the strict-toolban / procedural-
|
| 141 |
# compliance family. tools_called counts each tool name the agent
|
| 142 |
# invoked this episode; tool_violations counts how many of those calls
|
|
@@ -202,7 +202,52 @@ _PREDICATES: dict[str, Callable[[WinContext, Any], bool]] = {
|
|
| 202 |
}
|
| 203 |
|
| 204 |
LEAF_KEYS = frozenset(_PREDICATES)
|
| 205 |
-
COMPOSITE_KEYS = frozenset({"all_of", "any_of", "not"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
|
| 207 |
|
| 208 |
class WinCondition(BaseModel):
|
|
@@ -241,6 +286,8 @@ class WinCondition(BaseModel):
|
|
| 241 |
return any(WinCondition(**c).evaluate(ctx) for c in node["any_of"])
|
| 242 |
if "not" in node:
|
| 243 |
return not WinCondition(**node["not"]).evaluate(ctx)
|
|
|
|
|
|
|
| 244 |
return all(_PREDICATES[k](ctx, v) for k, v in node.items())
|
| 245 |
|
| 246 |
|
|
|
|
| 202 |
}
|
| 203 |
|
| 204 |
LEAF_KEYS = frozenset(_PREDICATES)
|
| 205 |
+
COMPOSITE_KEYS = frozenset({"all_of", "any_of", "not", "then"})
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def _then(ctx: WinContext, v: Any) -> bool:
|
| 209 |
+
"""Happened-before composite: each clause must become true IN ORDER
|
| 210 |
+
over the course of the episode. Stateful: the latch persists on
|
| 211 |
+
``signals.then_progress[id]`` so an early clause that stopped being
|
| 212 |
+
true still counts (analogous to ``waypoint_sequence``).
|
| 213 |
+
|
| 214 |
+
YAML form::
|
| 215 |
+
|
| 216 |
+
then:
|
| 217 |
+
id: scout-then-counter # stable key (per-episode)
|
| 218 |
+
clauses:
|
| 219 |
+
- {buildings_discovered_gte: 1} # FIRST
|
| 220 |
+
- {unit_type_count_gte: {type: e3, n: 4}} # THEN
|
| 221 |
+
|
| 222 |
+
Satisfied once every clause has been observed-true AT LEAST ONCE,
|
| 223 |
+
in order. Reaching a later clause without the earlier one being
|
| 224 |
+
latched does NOT advance — this is what makes "counter chosen
|
| 225 |
+
AFTER scout" enforceable (vs the stateless ``all_of`` which is
|
| 226 |
+
satisfied by any state where every clause happens to be true now).
|
| 227 |
+
"""
|
| 228 |
+
if not isinstance(v, dict):
|
| 229 |
+
raise ValueError("then: expects a dict with id + clauses")
|
| 230 |
+
clauses = v.get("clauses") or []
|
| 231 |
+
if not clauses:
|
| 232 |
+
return False
|
| 233 |
+
store = getattr(ctx.signals, "then_progress", None)
|
| 234 |
+
if store is None or not isinstance(store, dict):
|
| 235 |
+
store = {}
|
| 236 |
+
try:
|
| 237 |
+
ctx.signals.then_progress = store # type: ignore[attr-defined]
|
| 238 |
+
except Exception: # frozen/stub signals in unit tests
|
| 239 |
+
pass
|
| 240 |
+
key = str(v.get("id", id(v)))
|
| 241 |
+
idx = int(store.get(key, 0))
|
| 242 |
+
# Advance through every consecutive clause currently satisfied
|
| 243 |
+
# (typically ≤1 per evaluation, like waypoint_sequence).
|
| 244 |
+
while idx < len(clauses):
|
| 245 |
+
if WinCondition(**clauses[idx]).evaluate(ctx):
|
| 246 |
+
idx += 1
|
| 247 |
+
else:
|
| 248 |
+
break
|
| 249 |
+
store[key] = idx
|
| 250 |
+
return idx >= len(clauses)
|
| 251 |
|
| 252 |
|
| 253 |
class WinCondition(BaseModel):
|
|
|
|
| 286 |
return any(WinCondition(**c).evaluate(ctx) for c in node["any_of"])
|
| 287 |
if "not" in node:
|
| 288 |
return not WinCondition(**node["not"]).evaluate(ctx)
|
| 289 |
+
if "then" in node:
|
| 290 |
+
return _then(ctx, node["then"])
|
| 291 |
return all(_PREDICATES[k](ctx, v) for k, v in node.items())
|
| 292 |
|
| 293 |
|
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""then:[A,B,…] happened-before composite predicate.
|
| 2 |
+
|
| 3 |
+
Unblocks the reactive-replan / scout-then-counter / strict-ordered-
|
| 4 |
+
sequence family: a scenario can require clauses to become true IN
|
| 5 |
+
ORDER over the course of an episode, not merely all-of-now.
|
| 6 |
+
|
| 7 |
+
Real-world anchor: PlanBench replanning, ALFWorld goal-conditional
|
| 8 |
+
adaptation, PERT stage dependencies. The bench's existing predicates
|
| 9 |
+
let you say "all of these must be true now" (`all_of`) and "any of
|
| 10 |
+
these must be true now" (`any_of`); `then:` is the missing temporal
|
| 11 |
+
operator: the clauses must have been observed-true in sequence.
|
| 12 |
+
|
| 13 |
+
State: per-episode `signals.then_progress[id]` (an integer index of
|
| 14 |
+
how many clauses have been observed-true so far), so a clause that
|
| 15 |
+
stopped being true still counts. Identical pattern to
|
| 16 |
+
`waypoint_sequence`'s `seq_progress` latch.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
import types
|
| 22 |
+
|
| 23 |
+
from openra_bench.scenarios.win_conditions import (
|
| 24 |
+
COMPOSITE_KEYS,
|
| 25 |
+
WinContext,
|
| 26 |
+
evaluate,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _ctx(units_killed: int = 0, buildings_disc: int = 0,
|
| 31 |
+
own_units: int = 0):
|
| 32 |
+
sig = types.SimpleNamespace(
|
| 33 |
+
units_killed=units_killed,
|
| 34 |
+
enemy_buildings_seen_ids=set(range(buildings_disc)),
|
| 35 |
+
game_tick=0,
|
| 36 |
+
units_lost=0,
|
| 37 |
+
cash=0, resources=0, power_provided=0, power_drained=0,
|
| 38 |
+
own_building_types=set(), own_buildings=[],
|
| 39 |
+
enemies_seen_ids=set(),
|
| 40 |
+
explored_percent=0.0,
|
| 41 |
+
then_progress={},
|
| 42 |
+
seq_progress={},
|
| 43 |
+
)
|
| 44 |
+
return WinContext(
|
| 45 |
+
signals=sig,
|
| 46 |
+
render_state={"units_summary": [
|
| 47 |
+
{"id": i, "type": "e1", "cell_x": 0, "cell_y": 0}
|
| 48 |
+
for i in range(own_units)
|
| 49 |
+
]},
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def test_then_is_a_composite_key():
|
| 54 |
+
"""The new operator must register as a composite (not a leaf), so
|
| 55 |
+
the WinCondition validator accepts {then: …} alongside all_of /
|
| 56 |
+
any_of / not."""
|
| 57 |
+
assert "then" in COMPOSITE_KEYS
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def test_then_satisfied_when_clauses_observed_in_order():
|
| 61 |
+
"""A → B in order ⇒ true. The clauses don't both need to be true
|
| 62 |
+
NOW; they just need to have been observed-true at some past
|
| 63 |
+
evaluation, in order."""
|
| 64 |
+
cond = {"then": {
|
| 65 |
+
"id": "scout-then-counter",
|
| 66 |
+
"clauses": [
|
| 67 |
+
{"buildings_discovered_gte": 1}, # A
|
| 68 |
+
{"units_killed_gte": 3}, # B
|
| 69 |
+
],
|
| 70 |
+
}}
|
| 71 |
+
ctx = _ctx(units_killed=0, buildings_disc=0)
|
| 72 |
+
# Initially: neither clause true.
|
| 73 |
+
assert evaluate(cond, ctx) is False
|
| 74 |
+
|
| 75 |
+
# A becomes true: scouted 1 building.
|
| 76 |
+
ctx.signals.enemy_buildings_seen_ids = {1}
|
| 77 |
+
assert evaluate(cond, ctx) is False # A latched, B not yet
|
| 78 |
+
assert ctx.signals.then_progress["scout-then-counter"] == 1
|
| 79 |
+
|
| 80 |
+
# A is no longer true (lost vision) — but the latch holds.
|
| 81 |
+
ctx.signals.enemy_buildings_seen_ids = set()
|
| 82 |
+
ctx.signals.units_killed = 3 # B now true
|
| 83 |
+
assert evaluate(cond, ctx) is True
|
| 84 |
+
assert ctx.signals.then_progress["scout-then-counter"] == 2
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def test_then_no_credit_for_b_alone_without_a():
|
| 88 |
+
"""B true while A is still false ⇒ the chain does NOT advance.
|
| 89 |
+
The latch only counts clauses observed-true-in-order. Once A
|
| 90 |
+
finally becomes true on a later evaluation, the chain advances
|
| 91 |
+
through every consecutive currently-satisfied clause in one pass
|
| 92 |
+
(greedy advance, mirroring `waypoint_sequence`'s semantics). The
|
| 93 |
+
bench-relevant guarantee is that a policy which ONLY ever made B
|
| 94 |
+
true (never A) can never satisfy the chain — keeping the
|
| 95 |
+
happened-before signal honest."""
|
| 96 |
+
cond = {"then": {
|
| 97 |
+
"id": "ordered-x",
|
| 98 |
+
"clauses": [
|
| 99 |
+
{"buildings_discovered_gte": 1}, # A
|
| 100 |
+
{"units_killed_gte": 3}, # B
|
| 101 |
+
],
|
| 102 |
+
}}
|
| 103 |
+
ctx = _ctx()
|
| 104 |
+
# B true, A false → no progress.
|
| 105 |
+
ctx.signals.units_killed = 5
|
| 106 |
+
assert evaluate(cond, ctx) is False
|
| 107 |
+
assert ctx.signals.then_progress["ordered-x"] == 0
|
| 108 |
+
# Many more evals with B-only must STILL not advance (no false
|
| 109 |
+
# credit for the late-A case).
|
| 110 |
+
for _ in range(20):
|
| 111 |
+
assert evaluate(cond, ctx) is False
|
| 112 |
+
assert ctx.signals.then_progress["ordered-x"] == 0
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def test_then_late_a_then_b_credits_both_clauses():
|
| 116 |
+
"""Once A finally latches, the greedy advance picks up any
|
| 117 |
+
currently-satisfied later clauses in the same call. This matches
|
| 118 |
+
waypoint_sequence's semantics and is the right semantic: 'I
|
| 119 |
+
already had the counter ready; now that I've also scouted, the
|
| 120 |
+
chain is satisfied'."""
|
| 121 |
+
cond = {"then": {
|
| 122 |
+
"id": "late-a",
|
| 123 |
+
"clauses": [
|
| 124 |
+
{"buildings_discovered_gte": 1},
|
| 125 |
+
{"units_killed_gte": 3},
|
| 126 |
+
],
|
| 127 |
+
}}
|
| 128 |
+
ctx = _ctx()
|
| 129 |
+
ctx.signals.units_killed = 5 # B pre-satisfied
|
| 130 |
+
assert evaluate(cond, ctx) is False # but A blocks
|
| 131 |
+
ctx.signals.enemy_buildings_seen_ids = {1} # A becomes true
|
| 132 |
+
assert evaluate(cond, ctx) is True # chain completes
|
| 133 |
+
assert ctx.signals.then_progress["late-a"] == 2
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def test_then_three_clauses_strict_order():
|
| 137 |
+
"""Three-stage chain: A → B → C. Each stage must latch before the
|
| 138 |
+
next can advance. Skipping is impossible by construction."""
|
| 139 |
+
cond = {"then": {
|
| 140 |
+
"id": "abc",
|
| 141 |
+
"clauses": [
|
| 142 |
+
{"buildings_discovered_gte": 1},
|
| 143 |
+
{"units_killed_gte": 2},
|
| 144 |
+
{"own_units_gte": 4},
|
| 145 |
+
],
|
| 146 |
+
}}
|
| 147 |
+
ctx = _ctx()
|
| 148 |
+
# Skip straight to C: own_units=5 but no scout, no kills.
|
| 149 |
+
ctx.render_state["units_summary"] = [
|
| 150 |
+
{"id": i, "type": "e1", "cell_x": 0, "cell_y": 0} for i in range(5)
|
| 151 |
+
]
|
| 152 |
+
assert evaluate(cond, ctx) is False
|
| 153 |
+
assert ctx.signals.then_progress["abc"] == 0
|
| 154 |
+
|
| 155 |
+
# Now satisfy A: scout 1 building.
|
| 156 |
+
ctx.signals.enemy_buildings_seen_ids = {1}
|
| 157 |
+
assert evaluate(cond, ctx) is False
|
| 158 |
+
# Stage advanced to 1, but B (kills) is still 0.
|
| 159 |
+
assert ctx.signals.then_progress["abc"] == 1
|
| 160 |
+
|
| 161 |
+
# Now B: kill 2.
|
| 162 |
+
ctx.signals.units_killed = 2
|
| 163 |
+
# On this evaluation, B advances (idx 1→2), and C is already true
|
| 164 |
+
# (5 own units >= 4) so C also advances (idx 2→3). Returns true.
|
| 165 |
+
assert evaluate(cond, ctx) is True
|
| 166 |
+
assert ctx.signals.then_progress["abc"] == 3
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def test_then_empty_clauses_returns_false():
|
| 170 |
+
"""Authoring-hygiene: an empty or missing clauses list must not
|
| 171 |
+
accidentally satisfy the predicate (silent always-true would be a
|
| 172 |
+
no-cheat bar violation)."""
|
| 173 |
+
assert evaluate({"then": {"id": "empty", "clauses": []}}, _ctx()) is False
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def test_then_phrase_translation():
|
| 177 |
+
"""The objective_brief must render `then` as a plain-language
|
| 178 |
+
ordered chain so the model knows the order matters."""
|
| 179 |
+
from openra_bench.game_knowledge import objective_brief
|
| 180 |
+
|
| 181 |
+
brief = objective_brief(
|
| 182 |
+
"description",
|
| 183 |
+
{"then": {
|
| 184 |
+
"id": "scout-then-counter",
|
| 185 |
+
"clauses": [
|
| 186 |
+
{"buildings_discovered_gte": 1},
|
| 187 |
+
{"units_killed_gte": 3},
|
| 188 |
+
],
|
| 189 |
+
}},
|
| 190 |
+
None,
|
| 191 |
+
max_turns=30,
|
| 192 |
+
)
|
| 193 |
+
assert "in this exact order" in brief
|
| 194 |
+
assert "THEN" in brief
|
| 195 |
+
assert "spot ≥1 enemy buildings" in brief
|
| 196 |
+
assert "destroy ≥3 enemy units" in brief
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def test_then_id_isolation():
|
| 200 |
+
"""Two `then` predicates with different ids must latch
|
| 201 |
+
independently — necessary for a pack that uses multiple ordered
|
| 202 |
+
chains (e.g. a win clause + a fail clause both with their own
|
| 203 |
+
sequencing)."""
|
| 204 |
+
a = {"then": {"id": "A", "clauses": [{"units_killed_gte": 1}]}}
|
| 205 |
+
b = {"then": {"id": "B", "clauses": [{"units_killed_gte": 5}]}}
|
| 206 |
+
ctx = _ctx(units_killed=1)
|
| 207 |
+
assert evaluate(a, ctx) is True
|
| 208 |
+
assert evaluate(b, ctx) is False
|
| 209 |
+
assert ctx.signals.then_progress["A"] == 1
|
| 210 |
+
assert ctx.signals.then_progress["B"] == 0
|