Spaces:
Running on Zero
Running on Zero
agharsallah commited on
Commit Β·
ce159dc
1
Parent(s): a71301e
feat: Implement well-known typed fields for verdicts in output models
Browse files- Introduced `_well_known_specs` function to define typed fields for `winner` and `scores` in agent output.
- Updated `build_output_model` to handle new field types, allowing `winner` to be optional and `scores` to default to an empty map.
- Enhanced `json_instruction` to provide specific hints for well-known fields in the output format.
- Modified `FishbowlSession` to derive `winner_kind` and `winning_models` based on verdicts, supporting team-based outcomes.
- Added validation tests for new competition configurations and verdict handling, ensuring proper behavior for both agent and team winners.
- Created comprehensive tests for the new verdict validation logic, including re-ask scenarios and score normalization.
- config/agents/mystery-judge.yaml +5 -2
- config/agents/spy-host.yaml +4 -2
- config/scenarios/mystery-roots.yaml +4 -0
- config/scenarios/the-steeped.yaml +12 -0
- docs/adr/0029-structured-verdicts.md +225 -0
- docs/adr/0030-gpu-memory-snapshots-cold-start.md +121 -0
- docs/architecture/manifest-spec.md +6 -0
- docs/architecture/scenario-authoring.md +14 -0
- docs/architecture/structured-output.md +33 -0
- docs/schema/agent-manifest.md +4 -0
- docs/schema/events.md +38 -1
- docs/schema/scenario-config.md +39 -0
- modal/README.md +5 -0
- modal/catalogue.py +28 -1
- modal/docs/deploying.md +41 -0
- modal/service.py +178 -5
- src/agents/base.py +110 -12
- src/agents/handlers.py +45 -0
- src/core/conductor.py +11 -0
- src/core/config.py +75 -0
- src/core/registry.py +7 -0
- src/core/run_index.py +9 -0
- src/core/structured.py +66 -9
- src/models/provider.py +11 -2
- src/ui/fishbowl/session.py +27 -5
- tests/test_config.py +130 -3
- tests/test_run_history.py +80 -3
- tests/test_structured.py +87 -0
- tests/test_verdict_validation.py +370 -0
config/agents/mystery-judge.yaml
CHANGED
|
@@ -4,7 +4,10 @@ persona: >
|
|
| 4 |
You are the Mystery Judge. After reviewing the clues and debate, declare the most
|
| 5 |
likely explanation in one confident sentence. Start with 'Verdict:'. Choose the
|
| 6 |
most interesting, specific answer the evidence supports.
|
| 7 |
-
Also report your `mood` (one of: thinking, calm, lying, panic, smug, truth, gossip)
|
|
|
|
|
|
|
|
|
|
| 8 |
subscribes_to: []
|
| 9 |
may_emit:
|
| 10 |
- judge.verdict
|
|
@@ -16,6 +19,6 @@ memory:
|
|
| 16 |
use_salience: true
|
| 17 |
salience_top_k: 8
|
| 18 |
tools: []
|
| 19 |
-
output_extra_fields: [mood]
|
| 20 |
hue: 320
|
| 21 |
archetype: the mystery judge
|
|
|
|
| 4 |
You are the Mystery Judge. After reviewing the clues and debate, declare the most
|
| 5 |
likely explanation in one confident sentence. Start with 'Verdict:'. Choose the
|
| 6 |
most interesting, specific answer the evidence supports.
|
| 7 |
+
Also report your `mood` (one of: thinking, calm, lying, panic, smug, truth, gossip),
|
| 8 |
+
your `winner` (the exact on-stage name of the mind whose hypothesis your verdict
|
| 9 |
+
endorses), and a `scores` map from each mind's on-stage name to a 0β10 rating of how
|
| 10 |
+
much their contribution supported the answer.
|
| 11 |
subscribes_to: []
|
| 12 |
may_emit:
|
| 13 |
- judge.verdict
|
|
|
|
| 19 |
use_salience: true
|
| 20 |
salience_top_k: 8
|
| 21 |
tools: []
|
| 22 |
+
output_extra_fields: [mood, winner, scores]
|
| 23 |
hue: 320
|
| 24 |
archetype: the mystery judge
|
config/agents/spy-host.yaml
CHANGED
|
@@ -8,7 +8,9 @@ persona: >
|
|
| 8 |
dissect" or "let the clues be cast". Weigh who sat oddly: whose clue fit a different drink,
|
| 9 |
who slipped a tell, who over-covered. Begin exactly with "Verdict:" then name that one mind
|
| 10 |
and the single tell that gave them away, in one or two sentences. This is the final word.
|
| 11 |
-
Also report your `mood` (one of: thinking, calm, lying, panic, smug, truth, gossip)
|
|
|
|
|
|
|
| 12 |
subscribes_to: []
|
| 13 |
may_emit:
|
| 14 |
- judge.verdict
|
|
@@ -22,6 +24,6 @@ memory:
|
|
| 22 |
use_salience: true
|
| 23 |
salience_top_k: 8
|
| 24 |
tools: []
|
| 25 |
-
output_extra_fields: [mood]
|
| 26 |
hue: 300
|
| 27 |
archetype: the host
|
|
|
|
| 8 |
dissect" or "let the clues be cast". Weigh who sat oddly: whose clue fit a different drink,
|
| 9 |
who slipped a tell, who over-covered. Begin exactly with "Verdict:" then name that one mind
|
| 10 |
and the single tell that gave them away, in one or two sentences. This is the final word.
|
| 11 |
+
Also report your `mood` (one of: thinking, calm, lying, panic, smug, truth, gossip),
|
| 12 |
+
your `winner` (the exact on-stage name of the mind you accuse), and a `scores` map from
|
| 13 |
+
each mind's on-stage name to a 0β10 suspicion rating.
|
| 14 |
subscribes_to: []
|
| 15 |
may_emit:
|
| 16 |
- judge.verdict
|
|
|
|
| 24 |
use_salience: true
|
| 25 |
salience_top_k: 8
|
| 26 |
tools: []
|
| 27 |
+
output_extra_fields: [mood, winner, scores]
|
| 28 |
hue: 300
|
| 29 |
archetype: the host
|
config/scenarios/mystery-roots.yaml
CHANGED
|
@@ -14,6 +14,10 @@ cast:
|
|
| 14 |
- hypothesis-former
|
| 15 |
- devils-advocate
|
| 16 |
- mystery-judge
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
governor:
|
| 18 |
max_turns: 60
|
| 19 |
max_calls_per_turn: 16
|
|
|
|
| 14 |
- hypothesis-former
|
| 15 |
- devils-advocate
|
| 16 |
- mystery-judge
|
| 17 |
+
# A judged scenario (ADR-0029): no ground truth, so the judge's pick *is* the result β
|
| 18 |
+
# its `winner` field names the mind whose hypothesis the verdict endorses.
|
| 19 |
+
competition:
|
| 20 |
+
kind: judged
|
| 21 |
governor:
|
| 22 |
max_turns: 60
|
| 23 |
max_calls_per_turn: 16
|
config/scenarios/the-steeped.yaml
CHANGED
|
@@ -25,6 +25,18 @@ cast:
|
|
| 25 |
- spy-nil
|
| 26 |
- spy-ovo
|
| 27 |
- spy-host
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
governor:
|
| 29 |
max_turns: 2000
|
| 30 |
max_calls_per_turn: 16
|
|
|
|
| 25 |
- spy-nil
|
| 26 |
- spy-ovo
|
| 27 |
- spy-host
|
| 28 |
+
# A versus contest with ground truth (ADR-0029): spy-nil holds the near-twin word.
|
| 29 |
+
# The spy-host's verdict accuses a mind; the handler scores that accusation against
|
| 30 |
+
# these teams in code β the herd wins iff the accused really is the spy.
|
| 31 |
+
competition:
|
| 32 |
+
kind: versus
|
| 33 |
+
teams:
|
| 34 |
+
spy:
|
| 35 |
+
- spy-nil
|
| 36 |
+
herd:
|
| 37 |
+
- spy-cara
|
| 38 |
+
- spy-bex
|
| 39 |
+
- spy-ovo
|
| 40 |
governor:
|
| 41 |
max_turns: 2000
|
| 42 |
max_calls_per_turn: 16
|
docs/adr/0029-structured-verdicts.md
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ADR-0029: Structured Verdicts β Winners as Data, Not Prose
|
| 2 |
+
|
| 3 |
+
## Status
|
| 4 |
+
|
| 5 |
+
Proposed
|
| 6 |
+
|
| 7 |
+
## Context
|
| 8 |
+
|
| 9 |
+
A `judge.verdict` today is prose with a `mood`. The run lifecycle (ADR-0026) already
|
| 10 |
+
*wants* a machine-readable winner β `FishbowlSession.finalize` reads
|
| 11 |
+
`verdict.payload.get("winner")` and `RunSummary` carries `winner` / `winning_model` β
|
| 12 |
+
but nothing reliably populates that key: judges emit `{kind, text, mood}` and the
|
| 13 |
+
winner lives only in the sentence "Verdict: Bex slipped the tell." A leaderboard, a
|
| 14 |
+
shareable trace with a scoreboard, and any "who won on what model" claim all need the
|
| 15 |
+
winner as data.
|
| 16 |
+
|
| 17 |
+
Three constraints shape the design:
|
| 18 |
+
|
| 19 |
+
1. **The structured-output contract is string-typed.** `build_output_model`
|
| 20 |
+
(`src/core/structured.py`, ADR-0016) makes every `output_extra_fields` entry a
|
| 21 |
+
*required `str`*. A winner is an optional cast name; scores are a `dict[str, float]`.
|
| 22 |
+
The offline tolerant parser already passes arbitrary JSON keys through untouched,
|
| 23 |
+
so only the typed live model and the prompt instruction are string-bound.
|
| 24 |
+
2. **Agents never see scenario config.** `Registry.build_scenario` resolves the cast
|
| 25 |
+
*before* constructing the `Scenario`; a judge's handler has no view of which
|
| 26 |
+
scenario it serves, so it cannot know the cast to validate against or the team map
|
| 27 |
+
to attribute with β unless the registry injects that context (the same seam as
|
| 28 |
+
`agent.manifest = manifest`).
|
| 29 |
+
3. **Some scenarios have ground truth, some don't, some have neither.** In The Steeped
|
| 30 |
+
the engine *knows* the spy (`spy-nil`) β the judge's accusation can be scored by
|
| 31 |
+
code. In Mystery Roots there is no truth; the judge's pick *is* the answer. In
|
| 32 |
+
Thousand Token Wood nothing wins. One mechanism must serve all three without
|
| 33 |
+
hard-coding scenario names in the engine.
|
| 34 |
+
|
| 35 |
+
Prize relevance: a code-stamped scoreboard over an LLM verdict is the cleanest
|
| 36 |
+
"AI is load-bearing for judgment, code is load-bearing for bookkeeping" story
|
| 37 |
+
(Best Agent, Best Demo), and the enriched trace strengthens the Sharing-is-Caring
|
| 38 |
+
export (ADR-0026).
|
| 39 |
+
|
| 40 |
+
## Decision
|
| 41 |
+
|
| 42 |
+
Make the winner a first-class, validated payload field, derived by the layer that
|
| 43 |
+
actually knows it: the LLM where judgment is the product, the handler where ground
|
| 44 |
+
truth exists, and never at all where the scenario declares no competition. Five
|
| 45 |
+
additive pieces, `schema_version` stays 1 (ADR-0009).
|
| 46 |
+
|
| 47 |
+
### 1. Scenario competition contract (`CompetitionConfig`, extends ADR-0011)
|
| 48 |
+
|
| 49 |
+
`ScenarioConfig` gains an optional block, validated in `src/core/config.py`:
|
| 50 |
+
|
| 51 |
+
```yaml
|
| 52 |
+
competition:
|
| 53 |
+
kind: versus | judged | none # default none; absent block == none
|
| 54 |
+
teams: # versus only
|
| 55 |
+
spy: [spy-nil]
|
| 56 |
+
herd: [spy-cara, spy-bex, spy-ovo]
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
Validation rules (in `CompetitionConfig` + the existing
|
| 60 |
+
`WorldConfig._check_cast_references` validator):
|
| 61 |
+
|
| 62 |
+
- `teams` is permitted only when `kind: versus`, must be non-empty there, with
|
| 63 |
+
non-empty, mutually disjoint member lists.
|
| 64 |
+
- Every team member must be in the scenario's `cast`.
|
| 65 |
+
- **No team label may collide with an agent name** β `winner` carries either an agent
|
| 66 |
+
name or a team label, and this rule is what keeps that union unambiguous.
|
| 67 |
+
|
| 68 |
+
The Steeped declares `versus` with the teams above; Mystery Roots declares `judged`;
|
| 69 |
+
Thousand Token Wood declares nothing (`none`). `kind: none` scenarios keep full
|
| 70 |
+
sessions/history (ADR-0027) β they simply never produce a winner.
|
| 71 |
+
|
| 72 |
+
### 2. Well-known typed extra fields (extends ADR-0016)
|
| 73 |
+
|
| 74 |
+
`output_extra_fields` stays a `list[str]` β no manifest syntax change. Instead,
|
| 75 |
+
`src/core/structured.py` gains a small table of **well-known field types**:
|
| 76 |
+
|
| 77 |
+
| field | type | required |
|
| 78 |
+
|----------|---------------------|----------|
|
| 79 |
+
| `winner` | `str \| None` | no (default `None`) |
|
| 80 |
+
| `scores` | `dict[str, float]` | no (default `{}`) |
|
| 81 |
+
| *other* | `str` | yes (unchanged) |
|
| 82 |
+
|
| 83 |
+
`build_output_model` consults the table; `json_instruction` renders a typed schema
|
| 84 |
+
hint for known fields (`"winner": "<name or null>"`, `"scores": {"<name>": 0-10}`).
|
| 85 |
+
`winner` and `scores` are not arbitrary scenario fields β they are the verdict
|
| 86 |
+
contract ADR-0026 already names in `run.finished` β so giving them engine-known types
|
| 87 |
+
is the same move as `CORE_EVENT_KINDS`: open surface, curated core. Back-compat is
|
| 88 |
+
total: every existing manifest (`[mood]`, `[thought]`, β¦) hits the *other* row and
|
| 89 |
+
behaves exactly as before; the offline parser needs no change because it already
|
| 90 |
+
passes non-string values through.
|
| 91 |
+
|
| 92 |
+
### 3. Winner validation and one re-ask, in the base agent
|
| 93 |
+
|
| 94 |
+
Validation lives in `ManifestAgent`, not in per-judge handlers β `_resolve_payload`
|
| 95 |
+
owns the model call, so it is the only seam where a re-ask is one extra round-trip
|
| 96 |
+
instead of a re-architecture:
|
| 97 |
+
|
| 98 |
+
- The registry attaches the scenario's competition context when assembling a cast:
|
| 99 |
+
`agent.competition = cfg.competition` in `build_scenario`, plus the cast name list
|
| 100 |
+
and team labels (the *valid winner vocabulary*).
|
| 101 |
+
- A new overridable hook `_validate_payload(parsed) -> str | None` runs after the
|
| 102 |
+
structured call (and after the offline parse). The base implementation activates
|
| 103 |
+
only when `role == "judge"`, a competition with `kind != none` is attached, and
|
| 104 |
+
`winner` is in the agent's extra fields. A present-but-unknown `winner` (not a cast
|
| 105 |
+
name, not a team label) returns an error string; a missing/`None` winner is *not*
|
| 106 |
+
an error (the field is optional, and the offline stub never emits it β determinism
|
| 107 |
+
preserved).
|
| 108 |
+
- On error, `_resolve_payload` re-asks **once**: the same prompt plus a corrective
|
| 109 |
+
line naming the valid options. Token usage from both calls is *accumulated* into
|
| 110 |
+
`last_usage` so the governor (ADR-0013) meters the retry. On a second failure the
|
| 111 |
+
invalid `winner` is dropped and `payload["no_contest"] = true` is stamped β the
|
| 112 |
+
verdict text still ships (the drama survives), the leaderboard simply gets no row.
|
| 113 |
+
- `scores` is validated but never re-asked (it is garnish, not load-bearing): unknown
|
| 114 |
+
agent keys are dropped, values clamped to 0β10.
|
| 115 |
+
|
| 116 |
+
### 4. Ground truth belongs in code (the versus path)
|
| 117 |
+
|
| 118 |
+
For `kind: versus`, the LLM's `winner` field is its **accusation** (a cast name,
|
| 119 |
+
validated by Β§3). The scenario's handler β `SpyHost` is the template β then computes
|
| 120 |
+
the scoreboard after `super().act()`:
|
| 121 |
+
|
| 122 |
+
```
|
| 123 |
+
accused = payload.pop("winner") # the judge's pick, kept as payload["accused"]
|
| 124 |
+
correct = accused in competition.teams["spy"]
|
| 125 |
+
winner = "herd" if correct else "spy" # team label, stamped by code
|
| 126 |
+
payload |= {"accused": accused, "correct": correct, "winner": winner}
|
| 127 |
+
```
|
| 128 |
+
|
| 129 |
+
The team map comes from `competition.teams`, not from a handler constant β the
|
| 130 |
+
curated `_REVEAL` dict stays what it is (demo content: secrets and reveal drama),
|
| 131 |
+
while *who is on which team* is scenario config. Offline fallback: when the stub's
|
| 132 |
+
verdict carries no `winner`, the handler scans the verdict text for the first cast
|
| 133 |
+
name mentioned and treats it as the accusation; if none is found, `no_contest`. This
|
| 134 |
+
keeps the no-API-key demo producing a full stamped scoreboard, deterministically.
|
| 135 |
+
|
| 136 |
+
For `kind: judged` (Mystery Roots), no handler is needed: the validated `winner`
|
| 137 |
+
*is* the result β AI is load-bearing for the judgment itself. For `kind: none`,
|
| 138 |
+
judges do not declare `winner` in their manifests at all; `mischief-critic` keeps
|
| 139 |
+
`[mood]` (the Wood's reckoning records what became real β nobody wins it).
|
| 140 |
+
|
| 141 |
+
### 5. Attribution contract (extends ADR-0026)
|
| 142 |
+
|
| 143 |
+
`winner` is now an agent name (*judged*) or a team label (*versus*), so the
|
| 144 |
+
`run.finished` payload and `RunSummary` gain two additive keys:
|
| 145 |
+
|
| 146 |
+
```
|
| 147 |
+
"winner": str | None # unchanged β display name for the leaderboard row
|
| 148 |
+
"winner_kind": "agent" | "team" | None
|
| 149 |
+
"winning_models": list[str] # model_endpoint of the winner, or of every
|
| 150 |
+
# member of the winning team (None entries dropped)
|
| 151 |
+
```
|
| 152 |
+
|
| 153 |
+
`winning_model` keeps its exact current meaning β a single cast agent's model β and
|
| 154 |
+
is populated only when `winner_kind == "agent"`; it is `None` for team wins (never a
|
| 155 |
+
guess). `FishbowlSession.finalize` resolves the kind by checking the winner against
|
| 156 |
+
the run's cast map first, then the scenario's team labels. A future leaderboard
|
| 157 |
+
renders one row per finished run in a `kind != none` scenario: winner name,
|
| 158 |
+
`correct` badge where present, and the winning model(s). The UI itself is out of
|
| 159 |
+
scope here.
|
| 160 |
+
|
| 161 |
+
## Consequences
|
| 162 |
+
|
| 163 |
+
- **The verdict is data and drama at once.** `judge.verdict` carries
|
| 164 |
+
`winner`/`accused`/`correct`/`scores` machine-readably while `text` stays the
|
| 165 |
+
spoken ruling; `finalize`'s existing best-effort read starts actually working.
|
| 166 |
+
- **All additive.** No schema bump, no migration; old ledgers, old manifests, and the
|
| 167 |
+
string-typed extra-field behaviour are untouched. The offline stub emits no
|
| 168 |
+
`winner`, so the deterministic demo is byte-identical except where the SpyHost
|
| 169 |
+
text-scan stamps the scoreboard.
|
| 170 |
+
- **One re-ask is bounded cost.** At most one extra model call per verdict, metered
|
| 171 |
+
by the governor; the failure mode is a missing leaderboard row, never a missing
|
| 172 |
+
show ending.
|
| 173 |
+
- **The well-known field table is a curated list in engine code.** A scenario cannot
|
| 174 |
+
invent a new *typed* field from YAML alone β accepted: arbitrary fields remain
|
| 175 |
+
available as strings, and a handler can always derive structure from them. If a
|
| 176 |
+
third typed field ever appears, revisit a declarative field-spec syntax (see
|
| 177 |
+
alternatives).
|
| 178 |
+
- **Validation vocabulary is injected, not discovered.** Agents now carry a small
|
| 179 |
+
piece of scenario context (`competition`). This is a deliberate, single-attribute
|
| 180 |
+
seam mirroring `agent.manifest`; it does not give agents the scenario object.
|
| 181 |
+
- **Risk: usage accounting on re-ask.** `last_usage` must sum both calls or the
|
| 182 |
+
governor undercounts; this is an explicit acceptance criterion, not an afterthought.
|
| 183 |
+
- Prize impact: strengthens Best Agent / Best Demo (code-stamped scoreboard over
|
| 184 |
+
small-model judgment) and Sharing-is-Caring (self-scoring trace); the future
|
| 185 |
+
leaderboard it enables feeds Community Choice polish. No track is disqualified;
|
| 186 |
+
the β€32B constraint is untouched.
|
| 187 |
+
|
| 188 |
+
## Alternatives considered
|
| 189 |
+
|
| 190 |
+
- **Typed field descriptors in manifest YAML** (`output_extra_fields: [{name: scores,
|
| 191 |
+
type: score_map}]`). Maximally declarative, but adds a config surface and a union
|
| 192 |
+
schema for exactly two fields the engine already treats as core in ADR-0026.
|
| 193 |
+
Rejected as not the thinnest slice; the well-known table can grow into this later
|
| 194 |
+
without breaking `list[str]`.
|
| 195 |
+
- **Keep extra fields string-typed; handlers parse `winner`/`scores` out of strings.**
|
| 196 |
+
No engine change, but every judge handler re-implements parsing and the live path
|
| 197 |
+
loses validation-by-construction β the exact regression ADR-0016 exists to prevent.
|
| 198 |
+
- **Validate/re-ask in the conductor or per-handler.** The conductor never re-prompts
|
| 199 |
+
(it has no prompt), and per-handler re-ask duplicates the retry across
|
| 200 |
+
spy-host/mystery-judge and inverts the `super().act()` flow. The base-class hook is
|
| 201 |
+
the only seam that owns both the prompt and the provider.
|
| 202 |
+
- **Let the LLM declare the team winner in versus scenarios.** Simpler wiring, but
|
| 203 |
+
the model can be *wrong about its own conclusion's consequence* (naming the spy yet
|
| 204 |
+
declaring the spy won). Where truth exists, code stamps it β this is the
|
| 205 |
+
load-bearing split the feature exists to demonstrate.
|
| 206 |
+
- **A separate `judge.scored` event kind.** Keeps `judge.verdict` lean, but splits
|
| 207 |
+
one ruling across two events that every consumer must re-join, and the Fishbowl
|
| 208 |
+
curtain-fall already keys on the first `judge.verdict`. One enriched event wins.
|
| 209 |
+
- **Reuse `winner` for the accusation and skip `accused`.** Loses the LLM's actual
|
| 210 |
+
pick once the handler overwrites it; `accused` + `correct` is what makes the trace
|
| 211 |
+
auditable.
|
| 212 |
+
|
| 213 |
+
## References
|
| 214 |
+
|
| 215 |
+
- ADR-0009 (open/additive event kinds β new payload keys, no schema bump)
|
| 216 |
+
- ADR-0011 (declarative validatable config β `CompetitionConfig` joins `ScenarioConfig`)
|
| 217 |
+
- ADR-0016 (validated structured output β `build_output_model` typing extended)
|
| 218 |
+
- ADR-0026 (run lifecycle β `run.finished` winner contract extended with
|
| 219 |
+
`winner_kind` / `winning_models`)
|
| 220 |
+
- `src/core/structured.py` β well-known field types, typed `json_instruction`
|
| 221 |
+
- `src/core/config.py` β `CompetitionConfig`, cross-validation in `WorldConfig`
|
| 222 |
+
- `src/agents/base.py` β `_validate_payload` hook, single re-ask, usage accumulation
|
| 223 |
+
- `src/agents/handlers.py` β `SpyHost` ground-truth scoreboard
|
| 224 |
+
- `src/core/registry.py` β competition context injection in `build_scenario`
|
| 225 |
+
- `src/ui/fishbowl/session.py`, `src/core/run_index.py` β attribution resolution
|
docs/adr/0030-gpu-memory-snapshots-cold-start.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ADR-0030: GPU Memory Snapshots and Keep-Warm for Cold-Start Latency
|
| 2 |
+
|
| 3 |
+
## Status
|
| 4 |
+
|
| 5 |
+
Accepted (extends [ADR-0014 *Modal model serving*](0014-modal-model-serving.md),
|
| 6 |
+
[ADR-0019](0019-single-model-catalogue-no-cloud-path.md))
|
| 7 |
+
|
| 8 |
+
## Context
|
| 9 |
+
|
| 10 |
+
Every served model scales to zero (`min_containers=0`), so the first request to
|
| 11 |
+
an idle endpoint pays the full cold-start pipeline: container boot β weight
|
| 12 |
+
download/load β engine warmup (CUDA-graph capture, compile cache). On the bigger
|
| 13 |
+
models this is **minutes**, which hurts in two places that matter for the
|
| 14 |
+
hackathon:
|
| 15 |
+
|
| 16 |
+
- **The live demo.** A judge's first click should not stare at a spinner while a
|
| 17 |
+
14B model loads β the first 30 seconds are scored.
|
| 18 |
+
- **Iteration speed.** Every healthcheck or engine run against a cold workspace
|
| 19 |
+
pays the same multi-minute tax per model.
|
| 20 |
+
|
| 21 |
+
The mitigations we already had are blunt: `min_containers` removes cold starts
|
| 22 |
+
entirely but burns GPU-hours around the clock, and the shared `vllm-cache`
|
| 23 |
+
Volume only amortizes *compilation* β weight load and warmup are still paid by
|
| 24 |
+
every cold container.
|
| 25 |
+
|
| 26 |
+
Modal's answer to exactly this is **memory snapshots**: checkpoint a booted
|
| 27 |
+
container (CPU state, and with the alpha `enable_gpu_snapshot` flag, GPU state)
|
| 28 |
+
and restore it on later cold starts instead of re-initializing. Modal's
|
| 29 |
+
documented vLLM recipe pairs snapshots with vLLM **sleep mode**: warm the engine,
|
| 30 |
+
offload weights to host RAM (`POST /sleep?level=1`), snapshot, then reload
|
| 31 |
+
weights on restore (`POST /wake_up`). Restores land in seconds.
|
| 32 |
+
|
| 33 |
+
The catch: the recipe requires a *class-based* lifecycle (`@modal.enter(snap=True)`
|
| 34 |
+
for the snapshotted warmup, `@modal.enter(snap=False)` for post-restore wake),
|
| 35 |
+
while our serving path (ADR-0014) registers plain `@app.function` web servers.
|
| 36 |
+
And GPU snapshots are alpha, with real constraints: single-GPU only, the model's
|
| 37 |
+
vLLM build must support sleep mode, and host RAM must hold the offloaded weights.
|
| 38 |
+
|
| 39 |
+
## Decision
|
| 40 |
+
|
| 41 |
+
**1. Snapshots are a per-model catalogue flag, not a global switch.**
|
| 42 |
+
`ModelConfig.gpu_snapshot: bool = False` in `modal/catalogue.py`. When set,
|
| 43 |
+
`service.register_model()` dispatches to a class-based registrar
|
| 44 |
+
(`_register_snapshot_model`) implementing Modal's recipe verbatim:
|
| 45 |
+
|
| 46 |
+
- `@modal.enter(snap=True)` β start `vllm serve` (with `--enable-sleep-mode`),
|
| 47 |
+
wait for the port, run three warmup completions so compile/caching work lands
|
| 48 |
+
*inside* the snapshot, then `POST /sleep?level=1`. `startup_timeout` bounds
|
| 49 |
+
this whole phase (download + load + warmup + sleep).
|
| 50 |
+
- `@modal.enter(snap=False)` β `POST /wake_up` after every restore (it also
|
| 51 |
+
runs on the snapshot-creating boot itself, which simply resumes serving).
|
| 52 |
+
- `@modal.web_server(..., label=cfg.endpoint_name)` β a no-op method that
|
| 53 |
+
exposes the already-running vLLM port. The `label` pins the public URL to
|
| 54 |
+
`β¦--<app>-<endpoint_name>.modal.run`, byte-identical to the function path, so
|
| 55 |
+
clients, the engine catalogue (ADR-0019), and the DNS-label tests are
|
| 56 |
+
untouched.
|
| 57 |
+
- Image gains `VLLM_SERVER_DEV_MODE=1` (exposes the sleep/wake endpoints) and
|
| 58 |
+
`TORCHINDUCTOR_COMPILE_THREADS=1` (snapshot-safe compile), both scoped to
|
| 59 |
+
snapshot models.
|
| 60 |
+
|
| 61 |
+
Helpers used by the class are **nested closures, not module functions**: the
|
| 62 |
+
class ships via cloudpickle (`serialized=True`), which pickles closures by value
|
| 63 |
+
but module-level functions by reference β and the `service` module doesn't exist
|
| 64 |
+
inside the container. (Verified by round-tripping the pickled class with the
|
| 65 |
+
local modules removed.)
|
| 66 |
+
|
| 67 |
+
**2. Conservative initial casting.** Snapshots are on for the well-behaved
|
| 68 |
+
single-GPU, native-vLLM models the cast hits hardest β `nemotron-3-nano-4b`
|
| 69 |
+
(tiny), `minicpm-4-1-8b` (fast), `nemotron-cascade-14b` (Judge specialist) β and
|
| 70 |
+
deliberately **off** where the recipe is unproven or impossible:
|
| 71 |
+
|
| 72 |
+
| Model | Why not |
|
| 73 |
+
| --- | --- |
|
| 74 |
+
| Gemma 4 12B / 26B | Nightly vLLM + Transformers modeling backend; sleep mode unverified on that path. |
|
| 75 |
+
| Nemotron-3-Nano-30B | ~60GB BF16 weights won't fit host RAM during sleep level 1. |
|
| 76 |
+
| MiniCPM-o 4.5 | Omni-modal custom code path; kept conservative like its other knobs. |
|
| 77 |
+
|
| 78 |
+
Rolling back any model is a one-line `gpu_snapshot=False` β the plain function
|
| 79 |
+
path is untouched by this ADR.
|
| 80 |
+
|
| 81 |
+
**3. A deploy-time keep-warm switch for demo day.** `MODAL_LLM_KEEP_WARM=N`
|
| 82 |
+
(mirroring the existing `MODAL_LLM_REQUIRE_AUTH` / `MODAL_LLM_JSON_LOGS` idiom)
|
| 83 |
+
raises `min_containers` to N **for profile-bound models only** β the four tiers
|
| 84 |
+
the cast actually runs on. Specialists keep scale-to-zero. This is the
|
| 85 |
+
belt-and-braces for the hours around a live demo; snapshots are the everyday
|
| 86 |
+
path.
|
| 87 |
+
|
| 88 |
+
## Consequences
|
| 89 |
+
|
| 90 |
+
- Cold starts on snapshot models drop from minutes to seconds; scale-out under
|
| 91 |
+
burst gets the same benefit (every new container is a restore, not a boot).
|
| 92 |
+
- The serving layer now has two registration shapes (function vs. class). Both
|
| 93 |
+
are produced by the same loop from the same `ModelConfig`, and the dispatch is
|
| 94 |
+
one `if` β but anyone debugging a snapshot model must know the lifecycle runs
|
| 95 |
+
through `@modal.enter` hooks, not the function body.
|
| 96 |
+
- GPU snapshots are **Modal-alpha**. Known sharp edges we accept and guard:
|
| 97 |
+
snapshots are invalidated by image changes (safe β they rebuild), restores can
|
| 98 |
+
fail if Volume files used during snapshot are deleted, and the feature is
|
| 99 |
+
single-GPU only (all snapshot models are `tensor_parallel_size=1`).
|
| 100 |
+
- The alpha surface is also an **API-stability risk**: the recipe is verified
|
| 101 |
+
against the pinned Modal SDK (1.4.3 β `App.cls` accepts `serialized` /
|
| 102 |
+
`enable_memory_snapshot` / `experimental_options`; `modal.web_server` accepts
|
| 103 |
+
`label`), but `experimental_options={"enable_gpu_snapshot": True}` carries no
|
| 104 |
+
compatibility promise. Re-verify these kwargs on every SDK bump; the per-model
|
| 105 |
+
rollback (`gpu_snapshot=False`) restores the untouched plain path.
|
| 106 |
+
- The test suite covers registration, dispatch, and the cloudpickle round-trip,
|
| 107 |
+
but a snapshot *restore* can only be observed live. Validate each snapshot
|
| 108 |
+
model with `modal/healthcheck.py` after its first deploy β and again before
|
| 109 |
+
demo day β rather than trusting the recipe on paper.
|
| 110 |
+
- The snapshot warmup makes three tiny completions at deploy-boot time; with
|
| 111 |
+
auth enabled they authenticate via the same `VLLM_API_KEY` the secret injects,
|
| 112 |
+
so `MODAL_LLM_REQUIRE_AUTH=1` keeps working.
|
| 113 |
+
- `MODAL_LLM_KEEP_WARM` left on costs real GPU-hours β it is documented as a
|
| 114 |
+
demo-window switch, and the default deploy keeps scale-to-zero.
|
| 115 |
+
- First boot per snapshot model is slightly *slower* (warmup + sleep before
|
| 116 |
+
serving), which is the right trade: it runs once per image/config change, not
|
| 117 |
+
per cold start.
|
| 118 |
+
- Prize impact: this makes the Modal serving path credibly demo-ready
|
| 119 |
+
(Modal Awards) and defends the first-30-seconds bar that Best Demo and
|
| 120 |
+
Community Choice are scored on. The no-API-key deterministic stub is
|
| 121 |
+
untouched, so the on-stage fallback stays reproducible.
|
docs/architecture/manifest-spec.md
CHANGED
|
@@ -143,6 +143,12 @@ scenario shape agent output without engine edits. The Fishbowl cast uses
|
|
| 143 |
deterministic stub synthesises them offline so the mind-reader works with no API key
|
| 144 |
(ADR-0021).
|
| 145 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
### `hue` / `archetype`
|
| 147 |
Optional presentation metadata, consumed by the Fishbowl UI presenter and **ignored by
|
| 148 |
the engine** (ADR-0021). `hue` (0β360) colours the agent's mind on stage; `archetype`
|
|
|
|
| 143 |
deterministic stub synthesises them offline so the mind-reader works with no API key
|
| 144 |
(ADR-0021).
|
| 145 |
|
| 146 |
+
Fields are required strings by default, but two names are **well-known and
|
| 147 |
+
engine-typed** (ADR-0029): `winner` (`str | None`, optional) and `scores`
|
| 148 |
+
(`dict[str, float]`, optional). Judges in competition scenarios list them β
|
| 149 |
+
`output_extra_fields: [mood, winner, scores]` β to make the verdict machine-readable.
|
| 150 |
+
See [structured-output.md](structured-output.md#well-known-typed-fields).
|
| 151 |
+
|
| 152 |
### `hue` / `archetype`
|
| 153 |
Optional presentation metadata, consumed by the Fishbowl UI presenter and **ignored by
|
| 154 |
the engine** (ADR-0021). `hue` (0β360) colours the agent's mind on stage; `archetype`
|
docs/architecture/scenario-authoring.md
CHANGED
|
@@ -375,6 +375,19 @@ techniques:
|
|
| 375 |
the same "decorate the emitted event" move `FortuneTeller` uses for `omen`. Riding the
|
| 376 |
real ledger, the reveal scrubs and replays like any other event.
|
| 377 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 378 |
Offline, curated lines + a per-role mood bias in `src/models/provider.py` (`_STUB_*`,
|
| 379 |
keyed by agent name) give the bluff a coherent arc with no API key; live, a real small
|
| 380 |
model improvises from the personas. Either way the cast never calls each other β they
|
|
@@ -390,6 +403,7 @@ only post to the shared log, and the seam shows.
|
|
| 390 |
|---|---|---|
|
| 391 |
| a new cast on existing patterns | agent + scenario YAML | none |
|
| 392 |
| a hidden-role / secret-info game | secret in each `persona`; reveal via a verdict `handler` | none (handler in `agents/`) |
|
|
|
|
| 393 |
| a new event kind | just use it in `may_emit` | none |
|
| 394 |
| an agent that calls an existing tool | a `handler` + `tools:` grant | none (handler in `agents/`) |
|
| 395 |
| an agent that calls a *new* tool | register it in `builtins.py` + grant it | tool registration only |
|
|
|
|
| 375 |
the same "decorate the emitted event" move `FortuneTeller` uses for `omen`. Riding the
|
| 376 |
real ledger, the reveal scrubs and replays like any other event.
|
| 377 |
|
| 378 |
+
- **Ground truth gets a code-stamped scoreboard.** The scenario declares a
|
| 379 |
+
[`competition:` block](../schema/scenario-config.md#competition-who-can-win-and-who-decides)
|
| 380 |
+
(`kind: versus`, ADR-0029) naming the teams β `spy: [spy-nil]` vs the herd. The
|
| 381 |
+
judge's `winner` field (a well-known typed extra field, validated with one re-ask)
|
| 382 |
+
is its *accusation*; the `SpyHost` handler then scores it in code: the herd wins
|
| 383 |
+
iff the accused really is the spy, and the verdict payload gains `accused`,
|
| 384 |
+
`correct`, and a team-label `winner`. This is the load-bearing split β the model
|
| 385 |
+
provides the judgment drama, code stamps the bookkeeping where truth exists.
|
| 386 |
+
(Contrast `mystery-roots`, `kind: judged`: no ground truth, so the model's
|
| 387 |
+
validated `winner` *is* the result, no handler needed.) Offline, the handler
|
| 388 |
+
recovers the accusation from the verdict text, so the no-API-key demo still
|
| 389 |
+
produces a full deterministic scoreboard.
|
| 390 |
+
|
| 391 |
Offline, curated lines + a per-role mood bias in `src/models/provider.py` (`_STUB_*`,
|
| 392 |
keyed by agent name) give the bluff a coherent arc with no API key; live, a real small
|
| 393 |
model improvises from the personas. Either way the cast never calls each other β they
|
|
|
|
| 403 |
|---|---|---|
|
| 404 |
| a new cast on existing patterns | agent + scenario YAML | none |
|
| 405 |
| a hidden-role / secret-info game | secret in each `persona`; reveal via a verdict `handler` | none (handler in `agents/`) |
|
| 406 |
+
| a scenario that produces a *winner* | a `competition:` block; `winner` in the judge's `output_extra_fields` (+ a scoring `handler` if ground truth exists) | none (ADR-0029) |
|
| 407 |
| a new event kind | just use it in `may_emit` | none |
|
| 408 |
| an agent that calls an existing tool | a `handler` + `tools:` grant | none (handler in `agents/`) |
|
| 409 |
| an agent that calls a *new* tool | register it in `builtins.py` + grant it | tool registration only |
|
docs/architecture/structured-output.md
CHANGED
|
@@ -140,6 +140,9 @@ They're useful for:
|
|
| 140 |
- Routing decisions (e.g. "if emotion=desperate, escalate to judge")
|
| 141 |
- Downstream agent context (the Echo agent could read the emitting agent's "wants")
|
| 142 |
|
|
|
|
|
|
|
|
|
|
| 143 |
---
|
| 144 |
|
| 145 |
## Testing structured output
|
|
@@ -178,6 +181,36 @@ model literally cannot validate with a kind it isn't authorised to emit. The
|
|
| 178 |
function is pure Pydantic β no provider, no network β so it is unit-tested
|
| 179 |
directly and importable with the structured-output dependency absent.
|
| 180 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
### The structured call
|
| 182 |
|
| 183 |
`LiteLLMProvider.complete_structured(role, prompt, response_model)` wraps the
|
|
|
|
| 140 |
- Routing decisions (e.g. "if emotion=desperate, escalate to judge")
|
| 141 |
- Downstream agent context (the Echo agent could read the emitting agent's "wants")
|
| 142 |
|
| 143 |
+
Most extra fields are required strings, but two names are **well-known and
|
| 144 |
+
engine-typed** β see *Well-known typed fields* below.
|
| 145 |
+
|
| 146 |
---
|
| 147 |
|
| 148 |
## Testing structured output
|
|
|
|
| 181 |
function is pure Pydantic β no provider, no network β so it is unit-tested
|
| 182 |
directly and importable with the structured-output dependency absent.
|
| 183 |
|
| 184 |
+
### Well-known typed fields
|
| 185 |
+
|
| 186 |
+
`output_extra_fields` stays a plain `list[str]` β no manifest syntax change β but
|
| 187 |
+
`src/core/structured.py` carries a small table of names the engine knows how to
|
| 188 |
+
type (ADR-0029):
|
| 189 |
+
|
| 190 |
+
| field | type | required |
|
| 191 |
+
|----------|---------------------|---------------------|
|
| 192 |
+
| `winner` | `str \| None` | no (default `None`) |
|
| 193 |
+
| `scores` | `dict[str, float]` | no (default `{}`) |
|
| 194 |
+
| *other* | `str` | yes (unchanged) |
|
| 195 |
+
|
| 196 |
+
`winner` and `scores` are not arbitrary scenario fields β they are the verdict
|
| 197 |
+
contract that `run.finished` already names (ADR-0026), so giving them engine-known
|
| 198 |
+
types is the same move as `CORE_EVENT_KINDS`: open surface, curated core. A judge
|
| 199 |
+
manifest lists them like any extra field (`output_extra_fields: [mood, winner,
|
| 200 |
+
scores]` β see `config/agents/mystery-judge.yaml`).
|
| 201 |
+
|
| 202 |
+
Both halves of the contract honour the table. `build_output_model` makes the typed
|
| 203 |
+
fields optional with defaults, and `json_instruction` renders a typed schema hint
|
| 204 |
+
instead of the generic string slot β `"winner": "<a player's name, or null>"`,
|
| 205 |
+
`"scores": {"<player>": 0-10}` β so a small model knows it may answer `null`.
|
| 206 |
+
Back-compat is total: every existing manifest (`[mood]`, `[thought]`, β¦) hits the
|
| 207 |
+
*other* row and behaves exactly as before, and the tolerant offline parser already
|
| 208 |
+
passed non-string values through untouched.
|
| 209 |
+
|
| 210 |
+
What happens to a *validated-but-wrong* `winner` (a name outside the cast) is the
|
| 211 |
+
verdict-validation story β one re-ask, then `no_contest` β documented in
|
| 212 |
+
[events.md](../schema/events.md#verdict-and-run-payloads-adr-0029) and ADR-0029.
|
| 213 |
+
|
| 214 |
### The structured call
|
| 215 |
|
| 216 |
`LiteLLMProvider.complete_structured(role, prompt, response_model)` wraps the
|
docs/schema/agent-manifest.md
CHANGED
|
@@ -63,6 +63,10 @@ output_extra_fields: [] # extra payload fields the model is asked for, e.
|
|
| 63 |
- **`handler`** stays `null` for the common case (the generic `ManifestAgent`).
|
| 64 |
Set it to a key registered via `@register_handler` for agents that call tools or
|
| 65 |
need custom prompt logic; the YAML still supplies all declarative fields.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
- **`memory.*`** layers are pure views over the ledger β see
|
| 67 |
[memory-stack.md](../architecture/memory-stack.md).
|
| 68 |
|
|
|
|
| 63 |
- **`handler`** stays `null` for the common case (the generic `ManifestAgent`).
|
| 64 |
Set it to a key registered via `@register_handler` for agents that call tools or
|
| 65 |
need custom prompt logic; the YAML still supplies all declarative fields.
|
| 66 |
+
- **`output_extra_fields`** entries are required strings, except the well-known
|
| 67 |
+
typed names `winner` and `scores` (optional, engine-typed β ADR-0029). A judge in
|
| 68 |
+
a competition scenario lists them to make its verdict machine-readable; see
|
| 69 |
+
[structured-output.md](../architecture/structured-output.md#well-known-typed-fields).
|
| 70 |
- **`memory.*`** layers are pure views over the ledger β see
|
| 71 |
[memory-stack.md](../architecture/memory-stack.md).
|
| 72 |
|
docs/schema/events.md
CHANGED
|
@@ -36,7 +36,7 @@ kinds with **zero engine edits** β that is the modularity contract for the sch
|
|
| 36 |
memory importance defaults). It is a default set, not a gate:
|
| 37 |
|
| 38 |
```
|
| 39 |
-
run.started Β· world.observed Β· agent.thought Β· agent.spoke
|
| 40 |
agent.reflected Β· judge.verdict Β· user.injected
|
| 41 |
```
|
| 42 |
|
|
@@ -44,6 +44,43 @@ Any other well-formed kind (e.g. `oracle.spoke`, `crier.announced`) is valid and
|
|
| 44 |
if it carries a `text` payload, renders on stage via the generic projection
|
| 45 |
fallback.
|
| 46 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
## Evolution Rules
|
| 48 |
|
| 49 |
- Add fields instead of renaming fields; keep history immutable.
|
|
|
|
| 36 |
memory importance defaults). It is a default set, not a gate:
|
| 37 |
|
| 38 |
```
|
| 39 |
+
run.started Β· run.finished Β· world.observed Β· agent.thought Β· agent.spoke
|
| 40 |
agent.reflected Β· judge.verdict Β· user.injected
|
| 41 |
```
|
| 42 |
|
|
|
|
| 44 |
if it carries a `text` payload, renders on stage via the generic projection
|
| 45 |
fallback.
|
| 46 |
|
| 47 |
+
## Verdict and run payloads (ADR-0029)
|
| 48 |
+
|
| 49 |
+
In a scenario with a [`competition:` block](scenario-config.md#competition-who-can-win-and-who-decides),
|
| 50 |
+
the winner is data, not just prose. Two core kinds carry it β all keys additive,
|
| 51 |
+
`schema_version` stays 1 (ADR-0009).
|
| 52 |
+
|
| 53 |
+
### `judge.verdict`
|
| 54 |
+
|
| 55 |
+
Alongside the spoken `text` (and any manifest extras like `mood`):
|
| 56 |
+
|
| 57 |
+
| key | type | when present | meaning |
|
| 58 |
+
|---|---|---|---|
|
| 59 |
+
| `winner` | `str \| None` | `judged` and `versus` | agent name (*judged* β the model's validated pick) or team label (*versus* β stamped by code) |
|
| 60 |
+
| `accused` | `str` | `versus` | the judge's named pick, preserved before code overwrites `winner` β keeps the trace auditable |
|
| 61 |
+
| `correct` | `bool` | `versus` | ground truth: was the accused actually the spy? |
|
| 62 |
+
| `scores` | `dict[str, float]` | when in the judge's `output_extra_fields` | per-agent map, cleaned in code: unknown names dropped, values clamped to 0β10 |
|
| 63 |
+
| `no_contest` | `true` | on failure | the model named an invalid winner and one corrective re-ask didn't fix it; `winner` is dropped, the verdict `text` still ships |
|
| 64 |
+
|
| 65 |
+
An invalid `winner` (not a cast name, not a team label) triggers **one** re-ask in
|
| 66 |
+
`ManifestAgent` (`src/agents/base.py`), with both calls' token usage summed so the
|
| 67 |
+
governor meters the retry (ADR-0013). A missing `winner` is never an error β the
|
| 68 |
+
offline stub doesn't emit it, so deterministic demos are unaffected.
|
| 69 |
+
|
| 70 |
+
### `run.finished`
|
| 71 |
+
|
| 72 |
+
The attribution contract (ADR-0026, extended by ADR-0029) β mirrored on `RunSummary`:
|
| 73 |
+
|
| 74 |
+
| key | type | meaning |
|
| 75 |
+
|---|---|---|
|
| 76 |
+
| `winner` | `str \| None` | display name for the leaderboard row β agent name or team label |
|
| 77 |
+
| `winner_kind` | `"agent" \| "team" \| None` | how to read `winner`: checked against the run's cast map first, then team labels |
|
| 78 |
+
| `winning_model` | `str \| None` | unchanged legacy key β a single agent winner's `model_endpoint`; `None` for team wins (never a guess) |
|
| 79 |
+
| `winning_models` | `list[str]` | the winner's endpoint, or every winning-team member's (`None` entries dropped) |
|
| 80 |
+
|
| 81 |
+
`FishbowlSession.finalize` (`src/ui/fishbowl/session.py`) resolves the kind and
|
| 82 |
+
models when stamping `run.finished`.
|
| 83 |
+
|
| 84 |
## Evolution Rules
|
| 85 |
|
| 86 |
- Add fields instead of renaming fields; keep history immutable.
|
docs/schema/scenario-config.md
CHANGED
|
@@ -21,6 +21,8 @@ cast: # agent names, resolved via the agent registry
|
|
| 21 |
- devils-advocate
|
| 22 |
- mystery-judge
|
| 23 |
genesis_text: "A mystery settles over the wood: {seed}" # '{seed}' substituted
|
|
|
|
|
|
|
| 24 |
governor: # optional per-scenario budget (else defaults)
|
| 25 |
max_turns: 2000
|
| 26 |
max_calls_per_turn: 16
|
|
@@ -38,8 +40,45 @@ governor: # optional per-scenario budget (else defaults)
|
|
| 38 |
| `example_seeds` | Seed gallery for the UI dropdown. |
|
| 39 |
| `cast` | Agent names that participate. **Selecting who participates is editing this list.** Each must exist in `config/agents/`. |
|
| 40 |
| `genesis_text` | Template for the opening `world.observed`; `{seed}` is replaced. |
|
|
|
|
| 41 |
| `governor` | Optional `GovernorConfig`; omit for engine defaults. |
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
## Scheduling lives on the agents
|
| 44 |
|
| 45 |
A scenario does **not** declare a scheduling policy. Cadence is per-agent β
|
|
|
|
| 21 |
- devils-advocate
|
| 22 |
- mystery-judge
|
| 23 |
genesis_text: "A mystery settles over the wood: {seed}" # '{seed}' substituted
|
| 24 |
+
competition: # optional contest contract (ADR-0029); absent == none
|
| 25 |
+
kind: judged # versus | judged | none
|
| 26 |
governor: # optional per-scenario budget (else defaults)
|
| 27 |
max_turns: 2000
|
| 28 |
max_calls_per_turn: 16
|
|
|
|
| 40 |
| `example_seeds` | Seed gallery for the UI dropdown. |
|
| 41 |
| `cast` | Agent names that participate. **Selecting who participates is editing this list.** Each must exist in `config/agents/`. |
|
| 42 |
| `genesis_text` | Template for the opening `world.observed`; `{seed}` is replaced. |
|
| 43 |
+
| `competition` | Optional `CompetitionConfig` β does this scenario produce a winner, and how? See below. |
|
| 44 |
| `governor` | Optional `GovernorConfig`; omit for engine defaults. |
|
| 45 |
|
| 46 |
+
## Competition: who can win, and who decides
|
| 47 |
+
|
| 48 |
+
A scenario declares whether it produces a winner with the optional `competition:`
|
| 49 |
+
block (`CompetitionConfig`, ADR-0029). Absent block == `kind: none` β full sessions
|
| 50 |
+
and history, but nobody wins.
|
| 51 |
+
|
| 52 |
+
```yaml
|
| 53 |
+
competition:
|
| 54 |
+
kind: versus | judged | none # default none
|
| 55 |
+
teams: # versus only
|
| 56 |
+
spy: [spy-nil]
|
| 57 |
+
herd: [spy-cara, spy-bex, spy-ovo]
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
The three kinds split *who derives the winner*:
|
| 61 |
+
|
| 62 |
+
| kind | ground truth? | winner derived by | shipped example |
|
| 63 |
+
|---|---|---|---|
|
| 64 |
+
| `versus` | yes β the team map | **code** β the scenario's handler scores the judge's accusation against `teams` | `the-steeped` |
|
| 65 |
+
| `judged` | no β judgment *is* the result | **the model** β the judge's validated `winner` field | `mystery-roots` |
|
| 66 |
+
| `none` | n/a | nobody β judges don't declare `winner` at all | everything else |
|
| 67 |
+
|
| 68 |
+
Validation rules (enforced in `CompetitionConfig` and `WorldConfig`,
|
| 69 |
+
`src/core/config.py` β a bad block fails loudly at load, per ADR-0011):
|
| 70 |
+
|
| 71 |
+
- `teams` is permitted only when `kind: versus`, and is required (non-empty) there.
|
| 72 |
+
- Member lists must be non-empty and **mutually disjoint** β no double agents.
|
| 73 |
+
- Every team member must appear in the scenario's `cast`.
|
| 74 |
+
- **No team label may equal an agent name.** The `winner` payload key carries either
|
| 75 |
+
an agent name or a team label; this rule keeps that union unambiguous.
|
| 76 |
+
|
| 77 |
+
The registry injects the competition context into the cast's agents at build time
|
| 78 |
+
(the same seam as `agent.manifest`), which arms verdict validation in the base agent.
|
| 79 |
+
The machine-readable verdict and run-summary keys this produces are documented in
|
| 80 |
+
[events.md](events.md#verdict-and-run-payloads-adr-0029).
|
| 81 |
+
|
| 82 |
## Scheduling lives on the agents
|
| 83 |
|
| 84 |
A scenario does **not** declare a scheduling policy. Cadence is per-agent β
|
modal/README.md
CHANGED
|
@@ -71,6 +71,11 @@ sizing, and how to add models/providers or wire endpoints into the engine.
|
|
| 71 |
radius; one provider's outage or redeploy never touches another.
|
| 72 |
- **Scalable** β serverless autoscaling, input concurrency, a shared weight
|
| 73 |
cache (pull once, warm everywhere), and per-model `min_containers` warm pools.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
- **Extensible** β add a model = one `ModelConfig` in `catalogue.py`; add a
|
| 75 |
provider = one `Provider` entry + one app file. The serving path is written once
|
| 76 |
in `service.py`, and the engine picks up the new model with no edits (it reads
|
|
|
|
| 71 |
radius; one provider's outage or redeploy never touches another.
|
| 72 |
- **Scalable** β serverless autoscaling, input concurrency, a shared weight
|
| 73 |
cache (pull once, warm everywhere), and per-model `min_containers` warm pools.
|
| 74 |
+
- **Fast cold starts** β snapshot-enabled models (`gpu_snapshot=True`) restore a
|
| 75 |
+
pre-warmed engine from a Modal memory snapshot in seconds instead of re-paying
|
| 76 |
+
download + load + warmup; `MODAL_LLM_KEEP_WARM=1` at deploy time pins warm
|
| 77 |
+
containers for the tier models on demo day. See
|
| 78 |
+
[`docs/deploying.md` β Cold starts](docs/deploying.md#cold-starts) (ADR-0030).
|
| 79 |
- **Extensible** β add a model = one `ModelConfig` in `catalogue.py`; add a
|
| 80 |
provider = one `Provider` entry + one app file. The serving path is written once
|
| 81 |
in `service.py`, and the engine picks up the new model with no edits (it reads
|
modal/catalogue.py
CHANGED
|
@@ -82,6 +82,18 @@ class ModelConfig:
|
|
| 82 |
max_num_seqs: int | None = None # cap sequences batched per step (memory vs. throughput)
|
| 83 |
max_num_batched_tokens: int | None = None # token budget per scheduler step (prefill throughput)
|
| 84 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
# Observability / request logging (vLLM serve flags). Defaults give per-request
|
| 86 |
# visibility in the container logs out of the box; see ``service.build_command``.
|
| 87 |
log_requests: bool = True # log each request's id, sampling params, and token counts
|
|
@@ -153,12 +165,17 @@ NVIDIA_MODELS: tuple[ModelConfig, ...] = (
|
|
| 153 |
trust_remote_code=True,
|
| 154 |
gated=True,
|
| 155 |
max_concurrent_inputs=32,
|
|
|
|
|
|
|
|
|
|
| 156 |
),
|
| 157 |
ModelConfig(
|
| 158 |
name="nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
|
| 159 |
endpoint_name="nemotron-3-nano-30b",
|
| 160 |
# 30B total params in BF16 (~60GB) though only ~3B activate per token.
|
| 161 |
# An alternate strong model β not cast to a profile by default.
|
|
|
|
|
|
|
| 162 |
params_b=30,
|
| 163 |
gpu="H200:1",
|
| 164 |
max_model_len=32768,
|
|
@@ -187,6 +204,9 @@ NVIDIA_MODELS: tuple[ModelConfig, ...] = (
|
|
| 187 |
tool_call_parser="hermes",
|
| 188 |
enable_auto_tool_choice=True,
|
| 189 |
max_concurrent_inputs=48,
|
|
|
|
|
|
|
|
|
|
| 190 |
),
|
| 191 |
)
|
| 192 |
|
|
@@ -202,6 +222,10 @@ OPENBMB_MODELS: tuple[ModelConfig, ...] = (
|
|
| 202 |
max_model_len=32768,
|
| 203 |
trust_remote_code=True,
|
| 204 |
max_concurrent_inputs=48,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
# No tool_call_parser on purpose: MiniCPM4.1 emits a custom
|
| 206 |
# <|tool_call_start|> format vLLM 0.21.0 has no parser for, so tool-call
|
| 207 |
# structured output 400s here. The engine's structured path uses vLLM
|
|
@@ -252,7 +276,10 @@ GOOGLE_MODELS: tuple[ModelConfig, ...] = (
|
|
| 252 |
# Served via vLLM's Transformers modeling backend (gemma4_unified has no
|
| 253 |
# native vLLM class), which runs eager-only β CUDA-graph capture and the
|
| 254 |
# async scheduler aren't supported on that path, so disable both here.
|
| 255 |
-
# Prefix caching still applies and stays on (the default).
|
|
|
|
|
|
|
|
|
|
| 256 |
enforce_eager=True,
|
| 257 |
async_scheduling=False,
|
| 258 |
# Text-only in the cast (vision/audio is the MiniCPM-o specialist's job).
|
|
|
|
| 82 |
max_num_seqs: int | None = None # cap sequences batched per step (memory vs. throughput)
|
| 83 |
max_num_batched_tokens: int | None = None # token budget per scheduler step (prefill throughput)
|
| 84 |
|
| 85 |
+
# Cold starts. Opt a model into Modal memory snapshots (CPU + experimental GPU
|
| 86 |
+
# snapshot): the container boots once, loads weights, warms the engine, puts it
|
| 87 |
+
# to sleep (vLLM sleep mode, weights offloaded to host RAM), and is snapshotted;
|
| 88 |
+
# every later cold start restores the snapshot and wakes the engine in seconds
|
| 89 |
+
# instead of re-paying download + load + warmup. Constraints (why this is per
|
| 90 |
+
# model, not global): single-GPU models only, the model's vLLM build must
|
| 91 |
+
# support `--enable-sleep-mode`, and host RAM must hold the offloaded weights.
|
| 92 |
+
# Modal marks GPU snapshots alpha β keep it off for exotic serving paths
|
| 93 |
+
# (Transformers-backend Gemma, the omni specialist) and flip off on any model
|
| 94 |
+
# that misbehaves; the plain serving path is unchanged.
|
| 95 |
+
gpu_snapshot: bool = False
|
| 96 |
+
|
| 97 |
# Observability / request logging (vLLM serve flags). Defaults give per-request
|
| 98 |
# visibility in the container logs out of the box; see ``service.build_command``.
|
| 99 |
log_requests: bool = True # log each request's id, sampling params, and token counts
|
|
|
|
| 165 |
trust_remote_code=True,
|
| 166 |
gated=True,
|
| 167 |
max_concurrent_inputs=32,
|
| 168 |
+
# Tiny tier is the cast's hottest endpoint and 4B of BF16 weights (~8GB)
|
| 169 |
+
# easily fit host RAM during sleep β the ideal snapshot candidate.
|
| 170 |
+
gpu_snapshot=True,
|
| 171 |
),
|
| 172 |
ModelConfig(
|
| 173 |
name="nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
|
| 174 |
endpoint_name="nemotron-3-nano-30b",
|
| 175 |
# 30B total params in BF16 (~60GB) though only ~3B activate per token.
|
| 176 |
# An alternate strong model β not cast to a profile by default.
|
| 177 |
+
# No gpu_snapshot: sleep mode would offload ~60GB of weights to host RAM,
|
| 178 |
+
# past what a default container comfortably holds.
|
| 179 |
params_b=30,
|
| 180 |
gpu="H200:1",
|
| 181 |
max_model_len=32768,
|
|
|
|
| 204 |
tool_call_parser="hermes",
|
| 205 |
enable_auto_tool_choice=True,
|
| 206 |
max_concurrent_inputs=48,
|
| 207 |
+
# Qwen3-native single-GPU path on the pinned vLLM β snapshot-safe, and a
|
| 208 |
+
# reasoning model is exactly where a multi-minute cold start hurts most.
|
| 209 |
+
gpu_snapshot=True,
|
| 210 |
),
|
| 211 |
)
|
| 212 |
|
|
|
|
| 222 |
max_model_len=32768,
|
| 223 |
trust_remote_code=True,
|
| 224 |
max_concurrent_inputs=48,
|
| 225 |
+
# Fast tier default for the cast; 8B BF16 (~16GB) offloads to host RAM
|
| 226 |
+
# fine. Sleep mode is allocator-level, so the custom MiniCPM modeling
|
| 227 |
+
# code doesn't affect it.
|
| 228 |
+
gpu_snapshot=True,
|
| 229 |
# No tool_call_parser on purpose: MiniCPM4.1 emits a custom
|
| 230 |
# <|tool_call_start|> format vLLM 0.21.0 has no parser for, so tool-call
|
| 231 |
# structured output 400s here. The engine's structured path uses vLLM
|
|
|
|
| 276 |
# Served via vLLM's Transformers modeling backend (gemma4_unified has no
|
| 277 |
# native vLLM class), which runs eager-only β CUDA-graph capture and the
|
| 278 |
# async scheduler aren't supported on that path, so disable both here.
|
| 279 |
+
# Prefix caching still applies and stays on (the default). gpu_snapshot
|
| 280 |
+
# stays off too: sleep mode on the nightly Transformers backend is
|
| 281 |
+
# unverified, and the Gemmas already skip the costliest warmup (no
|
| 282 |
+
# CUDA-graph capture).
|
| 283 |
enforce_eager=True,
|
| 284 |
async_scheduling=False,
|
| 285 |
# Text-only in the cast (vision/audio is the MiniCPM-o specialist's job).
|
modal/docs/deploying.md
CHANGED
|
@@ -79,6 +79,7 @@ changes needed:
|
|
| 79 |
| `target_concurrent_inputs` | Autoscale target β scale out here, burst to the max (defaults to ~75% of the ceiling). |
|
| 80 |
| `buffer_containers` | Extra idle containers pre-warmed under active load (bursty traffic). |
|
| 81 |
| `scaledown_window` | Idle seconds before a container stops (cold-start vs. cost). |
|
|
|
|
| 82 |
| `min_containers` | Keep N warm to eliminate cold starts (always-on cost). |
|
| 83 |
| `gpu_memory_utilization` | Fraction of VRAM for weights + KV cache (vLLM default `0.9`); raise for a bigger KV cache. |
|
| 84 |
| `enable_prefix_caching` | Reuse the KV cache for shared prompt prefixes (on by default β big win when the system prompt / ledger context repeats across the cast). |
|
|
@@ -131,6 +132,46 @@ per model:
|
|
| 131 |
For memory-bound models, raise `gpu_memory_utilization` (more KV cache β more
|
| 132 |
concurrency) and cap `max_num_seqs` / `max_num_batched_tokens` if a step OOMs.
|
| 133 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
### Add a model
|
| 135 |
|
| 136 |
Append one `ModelConfig` to the appropriate provider list in `catalogue.py` (tag
|
|
|
|
| 79 |
| `target_concurrent_inputs` | Autoscale target β scale out here, burst to the max (defaults to ~75% of the ceiling). |
|
| 80 |
| `buffer_containers` | Extra idle containers pre-warmed under active load (bursty traffic). |
|
| 81 |
| `scaledown_window` | Idle seconds before a container stops (cold-start vs. cost). |
|
| 82 |
+
| `gpu_snapshot` | Serve via Modal memory snapshots (CPU + GPU): cold starts restore a warmed engine in seconds instead of re-paying load + warmup. See [Cold starts](#cold-starts). |
|
| 83 |
| `min_containers` | Keep N warm to eliminate cold starts (always-on cost). |
|
| 84 |
| `gpu_memory_utilization` | Fraction of VRAM for weights + KV cache (vLLM default `0.9`); raise for a bigger KV cache. |
|
| 85 |
| `enable_prefix_caching` | Reuse the KV cache for shared prompt prefixes (on by default β big win when the system prompt / ledger context repeats across the cast). |
|
|
|
|
| 132 |
For memory-bound models, raise `gpu_memory_utilization` (more KV cache β more
|
| 133 |
concurrency) and cap `max_num_seqs` / `max_num_batched_tokens` if a step OOMs.
|
| 134 |
|
| 135 |
+
### Cold starts
|
| 136 |
+
|
| 137 |
+
A scale-from-zero cold start normally pays the full pipeline: container boot β
|
| 138 |
+
weight load β engine warmup β minutes for the bigger models. Two mechanisms cut
|
| 139 |
+
this (ADR-0030):
|
| 140 |
+
|
| 141 |
+
**1. Memory snapshots (`gpu_snapshot=True`, per model).** The first container
|
| 142 |
+
boots once, loads weights, runs a few warmup completions, puts vLLM to sleep
|
| 143 |
+
(sleep level 1: weights offloaded to host RAM, KV cache dropped), and Modal
|
| 144 |
+
snapshots the container β CPU *and* GPU state. Every later cold start restores
|
| 145 |
+
the snapshot and wakes the engine, turning a multi-minute boot into seconds.
|
| 146 |
+
Under the hood this switches the model from the plain `@app.function` web server
|
| 147 |
+
to a class-based lifecycle (`@modal.enter(snap=True)` warmup β snapshot β
|
| 148 |
+
`@modal.enter(snap=False)` wake), but the public URL and API are identical β
|
| 149 |
+
clients can't tell the paths apart.
|
| 150 |
+
|
| 151 |
+
Snapshot-enabled today: `nemotron-3-nano-4b` (tiny), `minicpm-4-1-8b` (fast),
|
| 152 |
+
`nemotron-cascade-14b`. Left off deliberately: the Gemmas (nightly
|
| 153 |
+
Transformers-backend path, sleep mode unverified), `nemotron-3-nano-30b`
|
| 154 |
+
(~60GB of weights won't fit host RAM during sleep), and the omni specialist.
|
| 155 |
+
GPU snapshots are **Modal-alpha** β if a snapshot model misbehaves, set its
|
| 156 |
+
`gpu_snapshot=False` and redeploy; the plain path is unchanged.
|
| 157 |
+
|
| 158 |
+
**2. Demo-day keep-warm (deploy-time, no code edits).** Pin warm containers for
|
| 159 |
+
every *profile-bound* model (tiny/fast/balanced/strong) right before a live
|
| 160 |
+
demo β specialists keep scale-to-zero:
|
| 161 |
+
|
| 162 |
+
```bash
|
| 163 |
+
MODAL_LLM_KEEP_WARM=1 modal deploy modal/app_nvidia.py # one warm container per tier model
|
| 164 |
+
modal deploy modal/app_nvidia.py # back to scale-to-zero after
|
| 165 |
+
```
|
| 166 |
+
|
| 167 |
+
This burns GPU-hours while deployed; it's a switch for the hours around a demo,
|
| 168 |
+
not a steady state. `min_containers` in `catalogue.py` remains the per-model
|
| 169 |
+
override for anything finer-grained.
|
| 170 |
+
|
| 171 |
+
Cold-start clients must follow redirects: a Modal endpoint that hasn't answered
|
| 172 |
+
within ~150s returns a `303` to the same URL while the container finishes
|
| 173 |
+
booting (`modal/healthcheck.py` handles this; so does the engine's gateway).
|
| 174 |
+
|
| 175 |
### Add a model
|
| 176 |
|
| 177 |
Append one `ModelConfig` to the appropriate provider list in `catalogue.py` (tag
|
modal/service.py
CHANGED
|
@@ -80,6 +80,13 @@ JSON_LOGS = os.environ.get("MODAL_LLM_JSON_LOGS", "").lower() in ("1", "true", "
|
|
| 80 |
# config applies the same level). Read at deploy time and baked into the image.
|
| 81 |
LOG_LEVEL = os.environ.get("MODAL_LLM_LOG_LEVEL", "INFO").upper()
|
| 82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
# Where the structured-logging module + its generated config live in the
|
| 84 |
# container. The module dir goes on PYTHONPATH so vLLM can import the formatter
|
| 85 |
# the dictConfig references (``vllm_logging.JsonFormatter``).
|
|
@@ -139,6 +146,12 @@ def build_image(cfg: ModelConfig) -> modal.Image:
|
|
| 139 |
.env({"PYTHONPATH": _LOG_MODULE_DIR})
|
| 140 |
.env({"MODAL_LLM_JSON_LOGS": "1", "MODAL_LLM_LOG_LEVEL": LOG_LEVEL})
|
| 141 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
if cfg.extra_pip:
|
| 143 |
image = image.uv_pip_install(*cfg.extra_pip)
|
| 144 |
if cfg.env:
|
|
@@ -204,6 +217,10 @@ def build_command(cfg: ModelConfig) -> list[str]:
|
|
| 204 |
cmd += ["--tool-call-parser", cfg.tool_call_parser]
|
| 205 |
if cfg.mm_limits:
|
| 206 |
cmd += ["--limit-mm-per-prompt", json.dumps(cfg.mm_limits)]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
cmd += list(cfg.extra_vllm_args)
|
| 208 |
return cmd
|
| 209 |
|
|
@@ -211,12 +228,19 @@ def build_command(cfg: ModelConfig) -> list[str]:
|
|
| 211 |
# --- Endpoint registration ------------------------------------------------------
|
| 212 |
|
| 213 |
|
| 214 |
-
def register_model(app: modal.App, cfg: ModelConfig) -> modal.Function:
|
| 215 |
"""Attach one model to ``app`` as an autoscaling, OpenAI-compatible endpoint.
|
| 216 |
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
"""
|
| 221 |
image = build_image(cfg)
|
| 222 |
cmd = build_command(cfg)
|
|
@@ -227,11 +251,28 @@ def register_model(app: modal.App, cfg: ModelConfig) -> modal.Function:
|
|
| 227 |
# Exposes VLLM_API_KEY in the container; vLLM then enforces bearer auth.
|
| 228 |
secrets.append(modal.Secret.from_name(API_KEY_SECRET_NAME))
|
| 229 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
# Autoscale at the target, but let a hot container absorb a burst up to the
|
| 231 |
# hard max before another cold-starts (Modal high-perf-inference guidance).
|
| 232 |
# Default the target to ~75% of the ceiling so we scale out before saturating.
|
| 233 |
target_inputs = cfg.target_concurrent_inputs or max(1, (cfg.max_concurrent_inputs * 3) // 4)
|
| 234 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
function_kwargs = dict(
|
| 236 |
name=cfg.endpoint_name,
|
| 237 |
image=image,
|
|
@@ -239,7 +280,7 @@ def register_model(app: modal.App, cfg: ModelConfig) -> modal.Function:
|
|
| 239 |
volumes={HF_CACHE_PATH: hf_cache_vol, VLLM_CACHE_PATH: vllm_cache_vol},
|
| 240 |
secrets=secrets,
|
| 241 |
scaledown_window=cfg.scaledown_window,
|
| 242 |
-
min_containers=
|
| 243 |
timeout=cfg.request_timeout,
|
| 244 |
serialized=True,
|
| 245 |
)
|
|
@@ -270,6 +311,138 @@ def register_model(app: modal.App, cfg: ModelConfig) -> modal.Function:
|
|
| 270 |
return serve
|
| 271 |
|
| 272 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
def register_all(app: modal.App, configs: Iterable[ModelConfig]) -> None:
|
| 274 |
"""Register every model in ``configs`` onto ``app``."""
|
| 275 |
for cfg in configs:
|
|
|
|
| 80 |
# config applies the same level). Read at deploy time and baked into the image.
|
| 81 |
LOG_LEVEL = os.environ.get("MODAL_LLM_LOG_LEVEL", "INFO").upper()
|
| 82 |
|
| 83 |
+
# Demo-day switch: keep N containers warm for every *profile-bound* model (the
|
| 84 |
+
# tiers the cast actually runs on), removing their cold starts entirely for the
|
| 85 |
+
# duration of the deploy. Specialists keep scale-to-zero. Costs GPU-hours while
|
| 86 |
+
# deployed β turn it on right before a live demo, redeploy without it after:
|
| 87 |
+
# MODAL_LLM_KEEP_WARM=1 modal deploy modal/app_nvidia.py
|
| 88 |
+
KEEP_WARM = int(os.environ.get("MODAL_LLM_KEEP_WARM", "0") or "0")
|
| 89 |
+
|
| 90 |
# Where the structured-logging module + its generated config live in the
|
| 91 |
# container. The module dir goes on PYTHONPATH so vLLM can import the formatter
|
| 92 |
# the dictConfig references (``vllm_logging.JsonFormatter``).
|
|
|
|
| 146 |
.env({"PYTHONPATH": _LOG_MODULE_DIR})
|
| 147 |
.env({"MODAL_LLM_JSON_LOGS": "1", "MODAL_LLM_LOG_LEVEL": LOG_LEVEL})
|
| 148 |
)
|
| 149 |
+
if cfg.gpu_snapshot:
|
| 150 |
+
# Snapshot prerequisites: VLLM_SERVER_DEV_MODE exposes the /sleep and
|
| 151 |
+
# /wake_up endpoints the snapshot lifecycle drives, and single-threaded
|
| 152 |
+
# inductor compilation keeps torch.compile artifacts snapshot-safe
|
| 153 |
+
# (Modal's documented vLLM + GPU-snapshot recipe).
|
| 154 |
+
image = image.env({"VLLM_SERVER_DEV_MODE": "1", "TORCHINDUCTOR_COMPILE_THREADS": "1"})
|
| 155 |
if cfg.extra_pip:
|
| 156 |
image = image.uv_pip_install(*cfg.extra_pip)
|
| 157 |
if cfg.env:
|
|
|
|
| 217 |
cmd += ["--tool-call-parser", cfg.tool_call_parser]
|
| 218 |
if cfg.mm_limits:
|
| 219 |
cmd += ["--limit-mm-per-prompt", json.dumps(cfg.mm_limits)]
|
| 220 |
+
if cfg.gpu_snapshot:
|
| 221 |
+
# Sleep mode lets the snapshot lifecycle offload weights to host RAM
|
| 222 |
+
# (sleep level 1) before the memory snapshot is taken, then wake on restore.
|
| 223 |
+
cmd += ["--enable-sleep-mode"]
|
| 224 |
cmd += list(cfg.extra_vllm_args)
|
| 225 |
return cmd
|
| 226 |
|
|
|
|
| 228 |
# --- Endpoint registration ------------------------------------------------------
|
| 229 |
|
| 230 |
|
| 231 |
+
def register_model(app: modal.App, cfg: ModelConfig) -> modal.Function | type:
|
| 232 |
"""Attach one model to ``app`` as an autoscaling, OpenAI-compatible endpoint.
|
| 233 |
|
| 234 |
+
Dispatches on ``cfg.gpu_snapshot``: the default path is a serialized
|
| 235 |
+
``@app.function`` web server; snapshot models use a class-based lifecycle
|
| 236 |
+
(load β warm up β sleep β snapshot) so later cold starts restore in seconds
|
| 237 |
+
instead of re-paying download + load + warmup. Both paths publish the same
|
| 238 |
+
URL shape (``β¦--<app>-<endpoint_name>.modal.run``), so clients can't tell
|
| 239 |
+
them apart.
|
| 240 |
+
|
| 241 |
+
Everything is serialized (the prebuilt ``vllm serve`` argv is shipped to the
|
| 242 |
+
container), which lets us register many distinctly-named endpoints from a
|
| 243 |
+
simple loop without each needing a hand-written module-level function.
|
| 244 |
"""
|
| 245 |
image = build_image(cfg)
|
| 246 |
cmd = build_command(cfg)
|
|
|
|
| 251 |
# Exposes VLLM_API_KEY in the container; vLLM then enforces bearer auth.
|
| 252 |
secrets.append(modal.Secret.from_name(API_KEY_SECRET_NAME))
|
| 253 |
|
| 254 |
+
# Demo-day keep-warm: pin warm containers for the tier-bound models only β
|
| 255 |
+
# specialists keep scale-to-zero (see KEEP_WARM above).
|
| 256 |
+
min_containers = cfg.min_containers
|
| 257 |
+
if KEEP_WARM and cfg.profile:
|
| 258 |
+
min_containers = max(min_containers, KEEP_WARM)
|
| 259 |
+
|
| 260 |
# Autoscale at the target, but let a hot container absorb a burst up to the
|
| 261 |
# hard max before another cold-starts (Modal high-perf-inference guidance).
|
| 262 |
# Default the target to ~75% of the ceiling so we scale out before saturating.
|
| 263 |
target_inputs = cfg.target_concurrent_inputs or max(1, (cfg.max_concurrent_inputs * 3) // 4)
|
| 264 |
|
| 265 |
+
if cfg.gpu_snapshot:
|
| 266 |
+
return _register_snapshot_model(
|
| 267 |
+
app,
|
| 268 |
+
cfg,
|
| 269 |
+
image=image,
|
| 270 |
+
cmd=cmd,
|
| 271 |
+
secrets=secrets,
|
| 272 |
+
min_containers=min_containers,
|
| 273 |
+
target_inputs=target_inputs,
|
| 274 |
+
)
|
| 275 |
+
|
| 276 |
function_kwargs = dict(
|
| 277 |
name=cfg.endpoint_name,
|
| 278 |
image=image,
|
|
|
|
| 280 |
volumes={HF_CACHE_PATH: hf_cache_vol, VLLM_CACHE_PATH: vllm_cache_vol},
|
| 281 |
secrets=secrets,
|
| 282 |
scaledown_window=cfg.scaledown_window,
|
| 283 |
+
min_containers=min_containers,
|
| 284 |
timeout=cfg.request_timeout,
|
| 285 |
serialized=True,
|
| 286 |
)
|
|
|
|
| 311 |
return serve
|
| 312 |
|
| 313 |
|
| 314 |
+
def _class_name(slug: str) -> str:
|
| 315 |
+
"""Modal class name for an endpoint slug: ``nemotron-3-nano-4b`` β ``Nemotron3Nano4b``."""
|
| 316 |
+
return "".join(part.capitalize() for part in slug.replace("_", "-").split("-") if part) or "SnapshotServer"
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def _register_snapshot_model(
|
| 320 |
+
app: modal.App,
|
| 321 |
+
cfg: ModelConfig,
|
| 322 |
+
*,
|
| 323 |
+
image: modal.Image,
|
| 324 |
+
cmd: list[str],
|
| 325 |
+
secrets: list[modal.Secret],
|
| 326 |
+
min_containers: int,
|
| 327 |
+
target_inputs: int,
|
| 328 |
+
) -> type:
|
| 329 |
+
"""Snapshot serving path β Modal's vLLM + GPU-memory-snapshot recipe.
|
| 330 |
+
|
| 331 |
+
First boot: start vLLM, wait for the port, run a few warmup completions so
|
| 332 |
+
compiled artifacts and caches are resident, put the engine to sleep (weights
|
| 333 |
+
offloaded to host RAM, KV cache dropped), and let Modal snapshot the
|
| 334 |
+
container (CPU + GPU state). Every later cold start restores the snapshot
|
| 335 |
+
and wakes the engine β seconds instead of minutes. The web URL label is
|
| 336 |
+
pinned to ``cfg.endpoint_name`` so the public URL is identical to the plain
|
| 337 |
+
function path (``β¦--<app>-<endpoint_name>.modal.run``).
|
| 338 |
+
"""
|
| 339 |
+
served_name = cfg.served_name
|
| 340 |
+
|
| 341 |
+
# Helpers are nested (not module-level) on purpose: the class ships to the
|
| 342 |
+
# container via cloudpickle (``serialized=True``), and closures are pickled
|
| 343 |
+
# by value β a module-level helper would be pickled by reference to the
|
| 344 |
+
# ``service`` module, which doesn't exist inside the container.
|
| 345 |
+
def _headers() -> dict[str, str]:
|
| 346 |
+
import os
|
| 347 |
+
|
| 348 |
+
key = os.environ.get("VLLM_API_KEY")
|
| 349 |
+
return {"Authorization": f"Bearer {key}"} if key else {}
|
| 350 |
+
|
| 351 |
+
def _wait_ready(proc) -> None:
|
| 352 |
+
# vLLM opens the port only once the engine is initialized, so a
|
| 353 |
+
# successful connect means "ready", not just "listening".
|
| 354 |
+
import socket
|
| 355 |
+
import time
|
| 356 |
+
|
| 357 |
+
while True:
|
| 358 |
+
try:
|
| 359 |
+
socket.create_connection(("localhost", VLLM_PORT), timeout=1).close()
|
| 360 |
+
return
|
| 361 |
+
except OSError:
|
| 362 |
+
if proc.poll() is not None:
|
| 363 |
+
raise RuntimeError(f"vllm exited with code {proc.returncode}")
|
| 364 |
+
time.sleep(0.2)
|
| 365 |
+
|
| 366 |
+
def _post(path: str, json_body: dict | None = None, timeout: float = 300.0) -> None:
|
| 367 |
+
import requests # vLLM dependency, always present in the image
|
| 368 |
+
|
| 369 |
+
url = f"http://localhost:{VLLM_PORT}{path}"
|
| 370 |
+
requests.post(url, headers=_headers(), json=json_body, timeout=timeout).raise_for_status()
|
| 371 |
+
|
| 372 |
+
class _SnapshotServer:
|
| 373 |
+
@modal.enter(snap=True)
|
| 374 |
+
def start(self):
|
| 375 |
+
import os
|
| 376 |
+
import subprocess
|
| 377 |
+
|
| 378 |
+
env = dict(os.environ)
|
| 379 |
+
# Same structured-logging hook as the plain path (see ``serve``).
|
| 380 |
+
if env.get("MODAL_LLM_JSON_LOGS", "").lower() in ("1", "true", "yes"):
|
| 381 |
+
import vllm_logging
|
| 382 |
+
|
| 383 |
+
vllm_logging.write_config(_LOG_CONFIG_PATH, level=env.get("MODAL_LLM_LOG_LEVEL", "INFO"))
|
| 384 |
+
env["VLLM_LOGGING_CONFIG_PATH"] = _LOG_CONFIG_PATH
|
| 385 |
+
|
| 386 |
+
self.vllm_proc = subprocess.Popen(cmd, env=env)
|
| 387 |
+
_wait_ready(self.vllm_proc)
|
| 388 |
+
# Touch the full serving path so compile/caching work happens *before*
|
| 389 |
+
# the snapshot rather than on the first real request after restore.
|
| 390 |
+
warmup = {
|
| 391 |
+
"model": served_name,
|
| 392 |
+
"messages": [{"role": "user", "content": "Who tends the wood?"}],
|
| 393 |
+
"max_tokens": 8,
|
| 394 |
+
}
|
| 395 |
+
for _ in range(3):
|
| 396 |
+
_post("/v1/chat/completions", json_body=warmup)
|
| 397 |
+
# Offload weights to host RAM (sleep level 1); Modal snapshots the
|
| 398 |
+
# container right after the snap=True enters return.
|
| 399 |
+
_post("/sleep?level=1", timeout=120.0)
|
| 400 |
+
|
| 401 |
+
@modal.enter(snap=False)
|
| 402 |
+
def wake(self):
|
| 403 |
+
# Runs after every restore (and on the snapshot-creating boot itself,
|
| 404 |
+
# which simply resumes serving): reload weights onto the GPU.
|
| 405 |
+
_post("/wake_up", timeout=120.0)
|
| 406 |
+
_wait_ready(self.vllm_proc)
|
| 407 |
+
|
| 408 |
+
@modal.web_server(port=VLLM_PORT, startup_timeout=cfg.startup_timeout, label=cfg.endpoint_name)
|
| 409 |
+
def serve(self):
|
| 410 |
+
pass # vLLM (already running) is the web server; Modal just exposes the port.
|
| 411 |
+
|
| 412 |
+
@modal.exit()
|
| 413 |
+
def stop(self):
|
| 414 |
+
proc = getattr(self, "vllm_proc", None)
|
| 415 |
+
if proc is not None:
|
| 416 |
+
proc.terminate()
|
| 417 |
+
|
| 418 |
+
# One Modal class per model, named after the endpoint (App.cls has no name
|
| 419 |
+
# override, so rename the type before decorating).
|
| 420 |
+
name = _class_name(cfg.endpoint_name)
|
| 421 |
+
_SnapshotServer.__name__ = name
|
| 422 |
+
_SnapshotServer.__qualname__ = name
|
| 423 |
+
|
| 424 |
+
cls_kwargs = dict(
|
| 425 |
+
image=image,
|
| 426 |
+
gpu=cfg.gpu,
|
| 427 |
+
volumes={HF_CACHE_PATH: hf_cache_vol, VLLM_CACHE_PATH: vllm_cache_vol},
|
| 428 |
+
secrets=secrets,
|
| 429 |
+
scaledown_window=cfg.scaledown_window,
|
| 430 |
+
min_containers=min_containers,
|
| 431 |
+
timeout=cfg.request_timeout,
|
| 432 |
+
# Bounds the whole snap=True phase (download + load + warmup + sleep).
|
| 433 |
+
startup_timeout=cfg.startup_timeout,
|
| 434 |
+
serialized=True,
|
| 435 |
+
enable_memory_snapshot=True,
|
| 436 |
+
# GPU snapshots are Modal-alpha; scoped per model via cfg.gpu_snapshot.
|
| 437 |
+
experimental_options={"enable_gpu_snapshot": True},
|
| 438 |
+
)
|
| 439 |
+
if cfg.buffer_containers:
|
| 440 |
+
cls_kwargs["buffer_containers"] = cfg.buffer_containers
|
| 441 |
+
|
| 442 |
+
concurrent = modal.concurrent(max_inputs=cfg.max_concurrent_inputs, target_inputs=target_inputs)
|
| 443 |
+
return app.cls(**cls_kwargs)(concurrent(_SnapshotServer))
|
| 444 |
+
|
| 445 |
+
|
| 446 |
def register_all(app: modal.App, configs: Iterable[ModelConfig]) -> None:
|
| 447 |
"""Register every model in ``configs`` onto ``app``."""
|
| 448 |
for cfg in configs:
|
src/agents/base.py
CHANGED
|
@@ -115,6 +115,12 @@ class ManifestAgent(Agent):
|
|
| 115 |
self.memory_index = memory_index
|
| 116 |
self._reflection_tracker: ReflectionTracker | None = None
|
| 117 |
self.last_usage: dict[str, int] = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
# The model behind the most recent generation, captured when the provider is
|
| 119 |
# resolved and stamped onto the event in act()/reflection β so each line in the
|
| 120 |
# ledger records the model that actually produced it, not just the intended one.
|
|
@@ -241,6 +247,10 @@ class ManifestAgent(Agent):
|
|
| 241 |
Offline path (deterministic stub, no ``complete_structured``): append the
|
| 242 |
JSON instruction and run the tolerant parser as before. Token/cost usage
|
| 243 |
is recorded from the provider in every path.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
"""
|
| 245 |
wants_thought = bool(extra_fields and "thought" in extra_fields)
|
| 246 |
provider = self.router.for_profile(self._route_key)
|
|
@@ -248,25 +258,113 @@ class ManifestAgent(Agent):
|
|
| 248 |
with obs.span("agent.resolve", **{"mal.agent": role, "mal.profile": self._route_key}):
|
| 249 |
if hasattr(provider, "complete_structured"):
|
| 250 |
model = build_output_model(allowed, extra_fields)
|
| 251 |
-
|
| 252 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 253 |
self.last_usage = dict(provider.last_usage)
|
| 254 |
payload = self._with_reasoning(result.model_dump(), provider, "", wants_thought)
|
| 255 |
-
if is_usable_line(payload.get("text", ""))
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
|
|
|
|
|
|
| 260 |
obs.add_span_attrs(**{"resolve.path": "prose_fallback"})
|
| 261 |
return self._prose_fallback(role, prompt, allowed, wants_thought, provider)
|
| 262 |
|
| 263 |
instruction = json_instruction(allowed, extra_fields=extra_fields)
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
obs.add_span_attrs(**{"resolve.path": "offline_parse", "event.kind": parsed.get("kind", "")})
|
| 269 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 270 |
|
| 271 |
def _guard_model_error(self, role: str, raw: str) -> None:
|
| 272 |
"""Raise when *raw* is a provider failure sentinel, not a spoken line.
|
|
|
|
| 115 |
self.memory_index = memory_index
|
| 116 |
self._reflection_tracker: ReflectionTracker | None = None
|
| 117 |
self.last_usage: dict[str, int] = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
| 118 |
+
# Scenario competition context (ADR-0029), injected by the registry when this
|
| 119 |
+
# agent is assembled into a cast. ``None`` (the standalone default) means no
|
| 120 |
+
# competition: the verdict-validation hook is inert, so non-judged scenarios
|
| 121 |
+
# and bare-constructed agents behave exactly as before.
|
| 122 |
+
self.competition = None
|
| 123 |
+
self.cast_names: list[str] = []
|
| 124 |
# The model behind the most recent generation, captured when the provider is
|
| 125 |
# resolved and stamped onto the event in act()/reflection β so each line in the
|
| 126 |
# ledger records the model that actually produced it, not just the intended one.
|
|
|
|
| 247 |
Offline path (deterministic stub, no ``complete_structured``): append the
|
| 248 |
JSON instruction and run the tolerant parser as before. Token/cost usage
|
| 249 |
is recorded from the provider in every path.
|
| 250 |
+
|
| 251 |
+
On either path, a judge's verdict in a competition scenario is run through
|
| 252 |
+
:meth:`_verify_verdict` β one corrective re-ask when the model names a winner
|
| 253 |
+
outside the cast (ADR-0029), otherwise a no-op for every other agent.
|
| 254 |
"""
|
| 255 |
wants_thought = bool(extra_fields and "thought" in extra_fields)
|
| 256 |
provider = self.router.for_profile(self._route_key)
|
|
|
|
| 258 |
with obs.span("agent.resolve", **{"mal.agent": role, "mal.profile": self._route_key}):
|
| 259 |
if hasattr(provider, "complete_structured"):
|
| 260 |
model = build_output_model(allowed, extra_fields)
|
| 261 |
+
|
| 262 |
+
def _structured(p: str) -> dict | None:
|
| 263 |
+
"""One structured generation: a usable payload, or None to fall back."""
|
| 264 |
+
try:
|
| 265 |
+
result = provider.complete_structured(role, p, model)
|
| 266 |
+
except Exception:
|
| 267 |
+
self.last_usage = dict(provider.last_usage)
|
| 268 |
+
return None # structured failed β caller falls through to prose
|
| 269 |
self.last_usage = dict(provider.last_usage)
|
| 270 |
payload = self._with_reasoning(result.model_dump(), provider, "", wants_thought)
|
| 271 |
+
return payload if is_usable_line(payload.get("text", "")) else None
|
| 272 |
+
|
| 273 |
+
payload = _structured(prompt)
|
| 274 |
+
if payload is not None:
|
| 275 |
+
payload = self._verify_verdict(prompt, payload, _structured)
|
| 276 |
+
obs.add_span_attrs(**{"resolve.path": "structured", "event.kind": payload.get("kind", "")})
|
| 277 |
+
return payload
|
| 278 |
obs.add_span_attrs(**{"resolve.path": "prose_fallback"})
|
| 279 |
return self._prose_fallback(role, prompt, allowed, wants_thought, provider)
|
| 280 |
|
| 281 |
instruction = json_instruction(allowed, extra_fields=extra_fields)
|
| 282 |
+
|
| 283 |
+
def _offline(p: str) -> dict:
|
| 284 |
+
"""One offline generation: parse the stub's output into a payload."""
|
| 285 |
+
raw = provider.complete(role, f"{p}\n{instruction}")
|
| 286 |
+
self.last_usage = dict(provider.last_usage)
|
| 287 |
+
self._guard_model_error(role, raw)
|
| 288 |
+
parsed = parse_agent_output(raw, allowed_kinds=allowed, fallback_kind=allowed[0])
|
| 289 |
+
return self._with_reasoning(parsed, provider, raw, wants_thought)
|
| 290 |
+
|
| 291 |
+
parsed = self._verify_verdict(prompt, _offline(prompt), _offline)
|
| 292 |
obs.add_span_attrs(**{"resolve.path": "offline_parse", "event.kind": parsed.get("kind", "")})
|
| 293 |
+
return parsed
|
| 294 |
+
|
| 295 |
+
# ββ verdict winner validation (ADR-0029) βββββββββββββββββββββββββββββββββββ
|
| 296 |
+
|
| 297 |
+
def _verify_verdict(self, prompt: str, payload: dict, regenerate) -> dict:
|
| 298 |
+
"""Validate a verdict's ``winner``/``scores``; re-ask once on a bad winner.
|
| 299 |
+
|
| 300 |
+
``scores`` is normalised in place (non-cast keys dropped, values clamped to
|
| 301 |
+
0β10) and never re-asked β it is garnish. An out-of-cast ``winner`` triggers
|
| 302 |
+
exactly one corrective regeneration via *regenerate*, with the token usage of
|
| 303 |
+
both calls summed so the governor (ADR-0013) meters the retry. A second
|
| 304 |
+
failure drops ``winner`` and stamps ``no_contest`` β the verdict *text* still
|
| 305 |
+
ships, so the show always ends; only the leaderboard row is forfeited.
|
| 306 |
+
|
| 307 |
+
For every non-judge agent, every ``kind: none`` scenario, and every standalone
|
| 308 |
+
agent (no competition attached), :meth:`_validate_payload` returns ``None`` and
|
| 309 |
+
this is a transparent pass-through."""
|
| 310 |
+
error = self._validate_payload(payload)
|
| 311 |
+
if error is None:
|
| 312 |
+
return payload
|
| 313 |
+
first_usage = dict(self.last_usage)
|
| 314 |
+
corrective = (
|
| 315 |
+
f"\n\nCORRECTION: your previous reply named an invalid winner. {error} "
|
| 316 |
+
"Reply again with the same JSON object, changing only the 'winner' field."
|
| 317 |
+
)
|
| 318 |
+
retry = regenerate(prompt + corrective)
|
| 319 |
+
self.last_usage = self._sum_usage(first_usage, self.last_usage)
|
| 320 |
+
if retry is not None and is_usable_line(retry.get("text", "")) and self._validate_payload(retry) is None:
|
| 321 |
+
return retry
|
| 322 |
+
payload.pop("winner", None)
|
| 323 |
+
payload["no_contest"] = True
|
| 324 |
+
return payload
|
| 325 |
+
|
| 326 |
+
def _validate_payload(self, parsed: dict) -> str | None:
|
| 327 |
+
"""Return an error string when a judge named an out-of-cast winner, else ``None``.
|
| 328 |
+
|
| 329 |
+
Active only for a ``judge`` whose attached competition has ``kind != none`` and
|
| 330 |
+
whose manifest declares a ``winner`` field. A missing/empty ``winner`` is *not*
|
| 331 |
+
an error (the field is optional and the offline stub never emits it β determinism
|
| 332 |
+
preserved); ``scores`` is normalised in place as a side effect (never an error)."""
|
| 333 |
+
comp = self.competition
|
| 334 |
+
if comp is None or getattr(comp, "kind", "none") == "none" or self.manifest.role != "judge":
|
| 335 |
+
return None
|
| 336 |
+
if "winner" not in (self.manifest.output_extra_fields or []):
|
| 337 |
+
return None
|
| 338 |
+
cast = set(self.cast_names)
|
| 339 |
+
scores = parsed.get("scores")
|
| 340 |
+
if isinstance(scores, dict):
|
| 341 |
+
cleaned: dict[str, float] = {}
|
| 342 |
+
for name, value in scores.items():
|
| 343 |
+
if name not in cast:
|
| 344 |
+
continue
|
| 345 |
+
try:
|
| 346 |
+
cleaned[name] = max(0.0, min(10.0, float(value)))
|
| 347 |
+
except (TypeError, ValueError):
|
| 348 |
+
continue
|
| 349 |
+
parsed["scores"] = cleaned
|
| 350 |
+
winner = parsed.get("winner")
|
| 351 |
+
if winner in (None, ""):
|
| 352 |
+
return None
|
| 353 |
+
if winner not in self._winner_vocab():
|
| 354 |
+
return f"'winner' must be exactly one of: {', '.join(self._winner_vocab())}."
|
| 355 |
+
return None
|
| 356 |
+
|
| 357 |
+
def _winner_vocab(self) -> list[str]:
|
| 358 |
+
"""The valid winner vocabulary: every cast member, plus any team labels."""
|
| 359 |
+
comp = self.competition
|
| 360 |
+
teams = getattr(comp, "teams", None) or {}
|
| 361 |
+
return list(self.cast_names) + list(teams.keys())
|
| 362 |
+
|
| 363 |
+
@staticmethod
|
| 364 |
+
def _sum_usage(first: dict, second: dict) -> dict:
|
| 365 |
+
"""Sum two token-usage dicts so a re-ask's cost is metered, not lost."""
|
| 366 |
+
keys = set(first) | set(second)
|
| 367 |
+
return {k: int(first.get(k, 0) or 0) + int(second.get(k, 0) or 0) for k in keys}
|
| 368 |
|
| 369 |
def _guard_model_error(self, role: str, raw: str) -> None:
|
| 370 |
"""Raise when *raw* is a provider failure sentinel, not a spoken line.
|
src/agents/handlers.py
CHANGED
|
@@ -55,8 +55,53 @@ class SpyHost(ManifestAgent):
|
|
| 55 |
]
|
| 56 |
if reveal:
|
| 57 |
event.payload["reveal"] = reveal
|
|
|
|
| 58 |
return event
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
|
| 61 |
@register_handler("fortune-teller")
|
| 62 |
class FortuneTeller(ManifestAgent):
|
|
|
|
| 55 |
]
|
| 56 |
if reveal:
|
| 57 |
event.payload["reveal"] = reveal
|
| 58 |
+
self._stamp_scoreboard(event)
|
| 59 |
return event
|
| 60 |
|
| 61 |
+
def _stamp_scoreboard(self, event: Event) -> None:
|
| 62 |
+
"""Score the verdict in code β the load-bearing split of ADR-0029.
|
| 63 |
+
|
| 64 |
+
The judge's prose names a suspect (``payload['winner']`` on the live path);
|
| 65 |
+
this handler turns that *accusation* into the ground-truth *result* using the
|
| 66 |
+
scenario's ``competition.teams``: the herd wins when the named player really is
|
| 67 |
+
a spy, the spy wins otherwise. The accusation is preserved as ``accused`` and a
|
| 68 |
+
``correct`` flag rides alongside, so the trace stays auditable. Offline (no
|
| 69 |
+
``winner`` field) the accusation is recovered from the verdict text, so the
|
| 70 |
+
no-API-key demo still ends on a full, deterministic scoreboard. With no spy
|
| 71 |
+
team declared, or no recoverable accusation, the round is a ``no_contest``.
|
| 72 |
+
"""
|
| 73 |
+
comp = self.competition
|
| 74 |
+
spies = set((getattr(comp, "teams", None) or {}).get("spy", []))
|
| 75 |
+
if getattr(comp, "kind", "none") != "versus" or not spies:
|
| 76 |
+
return
|
| 77 |
+
accused = event.payload.get("winner") or self._scan_accusation(str(event.payload.get("text", "")))
|
| 78 |
+
if not accused:
|
| 79 |
+
event.payload.pop("winner", None)
|
| 80 |
+
event.payload["no_contest"] = True
|
| 81 |
+
return
|
| 82 |
+
correct = accused in spies
|
| 83 |
+
event.payload["accused"] = accused
|
| 84 |
+
event.payload["correct"] = correct
|
| 85 |
+
event.payload["winner"] = "herd" if correct else "spy"
|
| 86 |
+
|
| 87 |
+
def _scan_accusation(self, text: str) -> str | None:
|
| 88 |
+
"""Recover the accused player from verdict *text* β the first cast name named.
|
| 89 |
+
|
| 90 |
+
Matches each player by the distinctive tail of its agent name (``spy-cara`` β
|
| 91 |
+
``cara``), case-insensitively, and returns the one mentioned earliest. The host
|
| 92 |
+
itself is excluded so it never accuses the judge."""
|
| 93 |
+
low = text.lower()
|
| 94 |
+
best: str | None = None
|
| 95 |
+
best_at = len(low) + 1
|
| 96 |
+
for name in self.cast_names:
|
| 97 |
+
if name == self.name:
|
| 98 |
+
continue
|
| 99 |
+
token = name.split("-")[-1].lower()
|
| 100 |
+
at = low.find(token) if token else -1
|
| 101 |
+
if at != -1 and at < best_at:
|
| 102 |
+
best, best_at = name, at
|
| 103 |
+
return best
|
| 104 |
+
|
| 105 |
|
| 106 |
@register_handler("fortune-teller")
|
| 107 |
class FortuneTeller(ManifestAgent):
|
src/core/conductor.py
CHANGED
|
@@ -155,12 +155,20 @@ class Conductor:
|
|
| 155 |
*,
|
| 156 |
winner: str | None = None,
|
| 157 |
winning_model: str | None = None,
|
|
|
|
|
|
|
| 158 |
) -> Event | None:
|
| 159 |
"""Close the current run with a ``run.finished`` event.
|
| 160 |
|
| 161 |
Idempotent-safe: if this run already has a ``run.finished`` event we return
|
| 162 |
the existing one rather than emitting a duplicate. ``turns`` and ``tokens``
|
| 163 |
are read from the governor's live counters.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
"""
|
| 165 |
existing = [e for e in self.ledger.events_for_run(self.run_id) if e.kind == "run.finished"]
|
| 166 |
if existing:
|
|
@@ -174,7 +182,9 @@ class Conductor:
|
|
| 174 |
payload={
|
| 175 |
"reason": reason,
|
| 176 |
"winner": winner,
|
|
|
|
| 177 |
"winning_model": winning_model,
|
|
|
|
| 178 |
"turns": int(stats.get("current_turn", self.turn) or self.turn),
|
| 179 |
"tokens": int(stats.get("total_tokens", 0) or 0),
|
| 180 |
},
|
|
@@ -184,6 +194,7 @@ class Conductor:
|
|
| 184 |
run_id=self.run_id,
|
| 185 |
reason=reason,
|
| 186 |
winner=winner,
|
|
|
|
| 187 |
winning_model=winning_model,
|
| 188 |
turns=finished.payload["turns"],
|
| 189 |
tokens=finished.payload["tokens"],
|
|
|
|
| 155 |
*,
|
| 156 |
winner: str | None = None,
|
| 157 |
winning_model: str | None = None,
|
| 158 |
+
winner_kind: str | None = None,
|
| 159 |
+
winning_models: list[str] | None = None,
|
| 160 |
) -> Event | None:
|
| 161 |
"""Close the current run with a ``run.finished`` event.
|
| 162 |
|
| 163 |
Idempotent-safe: if this run already has a ``run.finished`` event we return
|
| 164 |
the existing one rather than emitting a duplicate. ``turns`` and ``tokens``
|
| 165 |
are read from the governor's live counters.
|
| 166 |
+
|
| 167 |
+
Attribution (ADR-0029): ``winner`` is a cast agent name (``winner_kind:
|
| 168 |
+
"agent"``) or a team label (``winner_kind: "team"``). ``winning_model`` keeps
|
| 169 |
+
its original meaning β a single cast agent's endpoint, populated only for an
|
| 170 |
+
agent winner β while ``winning_models`` lists the endpoint(s) behind the
|
| 171 |
+
winner (every member of a winning team). All keys are additive.
|
| 172 |
"""
|
| 173 |
existing = [e for e in self.ledger.events_for_run(self.run_id) if e.kind == "run.finished"]
|
| 174 |
if existing:
|
|
|
|
| 182 |
payload={
|
| 183 |
"reason": reason,
|
| 184 |
"winner": winner,
|
| 185 |
+
"winner_kind": winner_kind,
|
| 186 |
"winning_model": winning_model,
|
| 187 |
+
"winning_models": list(winning_models or []),
|
| 188 |
"turns": int(stats.get("current_turn", self.turn) or self.turn),
|
| 189 |
"tokens": int(stats.get("total_tokens", 0) or 0),
|
| 190 |
},
|
|
|
|
| 194 |
run_id=self.run_id,
|
| 195 |
reason=reason,
|
| 196 |
winner=winner,
|
| 197 |
+
winner_kind=winner_kind,
|
| 198 |
winning_model=winning_model,
|
| 199 |
turns=finished.payload["turns"],
|
| 200 |
tokens=finished.payload["tokens"],
|
src/core/config.py
CHANGED
|
@@ -16,8 +16,11 @@ to "emit JSON, validate it, run it." See ADR-0011.
|
|
| 16 |
The agent schema itself is :class:`AgentManifest` (``src/core/manifest.py``) β we
|
| 17 |
reuse it here rather than duplicating, so the four stable contracts stay singular.
|
| 18 |
"""
|
|
|
|
| 19 |
from __future__ import annotations
|
| 20 |
|
|
|
|
|
|
|
| 21 |
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
| 22 |
|
| 23 |
from src.core.manifest import AgentManifest
|
|
@@ -75,6 +78,56 @@ class GovernorConfig(BaseModel):
|
|
| 75 |
hourly_budget_usd: float | None = None
|
| 76 |
|
| 77 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
# ββ scenario βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 79 |
|
| 80 |
|
|
@@ -96,6 +149,9 @@ class ScenarioConfig(BaseModel):
|
|
| 96 |
genesis_text: str | None = None
|
| 97 |
governor: GovernorConfig | None = None
|
| 98 |
|
|
|
|
|
|
|
|
|
|
| 99 |
|
| 100 |
# ββ the whole world ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 101 |
|
|
@@ -125,6 +181,25 @@ class WorldConfig(BaseModel):
|
|
| 125 |
f"scenario {scenario.name!r} references undefined agents: {missing}. "
|
| 126 |
f"Defined agents: {sorted(defined)}"
|
| 127 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
return self
|
| 129 |
|
| 130 |
|
|
|
|
| 16 |
The agent schema itself is :class:`AgentManifest` (``src/core/manifest.py``) β we
|
| 17 |
reuse it here rather than duplicating, so the four stable contracts stay singular.
|
| 18 |
"""
|
| 19 |
+
|
| 20 |
from __future__ import annotations
|
| 21 |
|
| 22 |
+
from typing import Literal
|
| 23 |
+
|
| 24 |
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
| 25 |
|
| 26 |
from src.core.manifest import AgentManifest
|
|
|
|
| 78 |
hourly_budget_usd: float | None = None
|
| 79 |
|
| 80 |
|
| 81 |
+
# ββ competition ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class CompetitionConfig(BaseModel):
|
| 85 |
+
"""Declares whether β and how β a scenario produces a winner (ADR-0029).
|
| 86 |
+
|
| 87 |
+
A scenario can be a ``versus`` contest between named teams, a ``judged`` pick
|
| 88 |
+
where the judge's verdict *is* the result, or ``none`` (the default) where
|
| 89 |
+
nobody wins. ``winner`` downstream carries either an agent name or a team
|
| 90 |
+
label, so the team labels here must stay distinct from agent names β that
|
| 91 |
+
cross-cast check lives in :meth:`WorldConfig._check_cast_references`, while the
|
| 92 |
+
rules a competition can enforce on its own (team shape, disjointness) live in
|
| 93 |
+
the validator below.
|
| 94 |
+
"""
|
| 95 |
+
|
| 96 |
+
model_config = ConfigDict(extra="forbid")
|
| 97 |
+
|
| 98 |
+
kind: Literal["versus", "judged", "none"] = "none"
|
| 99 |
+
"""How a winner is derived β ``versus`` (team contest), ``judged`` (the judge's
|
| 100 |
+
pick is the answer), or ``none`` (no winner; the default and the absent block)."""
|
| 101 |
+
|
| 102 |
+
teams: dict[str, list[str]] | None = None
|
| 103 |
+
"""Team label β member agent names. Permitted only when ``kind == 'versus'``."""
|
| 104 |
+
|
| 105 |
+
@model_validator(mode="after")
|
| 106 |
+
def _check_teams(self) -> "CompetitionConfig":
|
| 107 |
+
if self.kind != "versus":
|
| 108 |
+
if self.teams is not None:
|
| 109 |
+
raise ValueError(f"competition.teams is only allowed when kind is 'versus' (got kind={self.kind!r})")
|
| 110 |
+
return self
|
| 111 |
+
# kind == "versus": teams are required and must describe a real contest.
|
| 112 |
+
if not self.teams:
|
| 113 |
+
raise ValueError("competition.kind 'versus' requires a non-empty 'teams' mapping")
|
| 114 |
+
empty = [label for label, members in self.teams.items() if not members]
|
| 115 |
+
if empty:
|
| 116 |
+
raise ValueError(f"competition.teams has empty member lists for teams: {sorted(empty)}")
|
| 117 |
+
seen: dict[str, str] = {}
|
| 118 |
+
overlap: set[str] = set()
|
| 119 |
+
for label, members in self.teams.items():
|
| 120 |
+
for member in members:
|
| 121 |
+
if member in seen and seen[member] != label:
|
| 122 |
+
overlap.add(member)
|
| 123 |
+
seen[member] = label
|
| 124 |
+
if overlap:
|
| 125 |
+
raise ValueError(
|
| 126 |
+
f"competition.teams must be mutually disjoint; agents on more than one team: {sorted(overlap)}"
|
| 127 |
+
)
|
| 128 |
+
return self
|
| 129 |
+
|
| 130 |
+
|
| 131 |
# ββ scenario βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 132 |
|
| 133 |
|
|
|
|
| 149 |
genesis_text: str | None = None
|
| 150 |
governor: GovernorConfig | None = None
|
| 151 |
|
| 152 |
+
competition: CompetitionConfig | None = None
|
| 153 |
+
"""Optional winner contract (ADR-0029); absent == ``none`` (no winner)."""
|
| 154 |
+
|
| 155 |
|
| 156 |
# ββ the whole world ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 157 |
|
|
|
|
| 181 |
f"scenario {scenario.name!r} references undefined agents: {missing}. "
|
| 182 |
f"Defined agents: {sorted(defined)}"
|
| 183 |
)
|
| 184 |
+
competition = scenario.competition
|
| 185 |
+
if competition is None or competition.teams is None:
|
| 186 |
+
continue
|
| 187 |
+
# Every team member must be in this scenario's cast (ADR-0029 Β§1).
|
| 188 |
+
cast = set(scenario.cast)
|
| 189 |
+
off_cast = sorted({m for members in competition.teams.values() for m in members if m not in cast})
|
| 190 |
+
if off_cast:
|
| 191 |
+
raise ValueError(
|
| 192 |
+
f"scenario {scenario.name!r} competition team members not in its cast: {off_cast}. "
|
| 193 |
+
f"Cast: {sorted(cast)}"
|
| 194 |
+
)
|
| 195 |
+
# A team label must not collide with any agent name, or the winner union
|
| 196 |
+
# (agent name OR team label) becomes ambiguous (ADR-0029 Β§1).
|
| 197 |
+
collisions = sorted(label for label in competition.teams if label in defined)
|
| 198 |
+
if collisions:
|
| 199 |
+
raise ValueError(
|
| 200 |
+
f"scenario {scenario.name!r} competition team labels collide with agent names: {collisions}. "
|
| 201 |
+
f"Team labels must be distinct from agent names to keep the winner unambiguous."
|
| 202 |
+
)
|
| 203 |
return self
|
| 204 |
|
| 205 |
|
src/core/registry.py
CHANGED
|
@@ -239,6 +239,13 @@ class Registry:
|
|
| 239 |
|
| 240 |
memory_index = memory_index_from_env()
|
| 241 |
agents = tuple(self.build_agent(agent_name, router, tools, memory_index) for agent_name in cfg.cast)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
obs.log(
|
| 243 |
"registry.cast_assembled",
|
| 244 |
scenario=cfg.name,
|
|
|
|
| 239 |
|
| 240 |
memory_index = memory_index_from_env()
|
| 241 |
agents = tuple(self.build_agent(agent_name, router, tools, memory_index) for agent_name in cfg.cast)
|
| 242 |
+
# Inject the scenario's competition context (ADR-0029) β the same single-attribute
|
| 243 |
+
# seam as ``agent.manifest``. This is the only scenario-level fact an agent sees:
|
| 244 |
+
# it lets a judge validate its winner against the cast and a versus handler attribute
|
| 245 |
+
# the win to a team. Absent block == no competition, so the hook stays inert.
|
| 246 |
+
for agent in agents:
|
| 247 |
+
agent.competition = cfg.competition
|
| 248 |
+
agent.cast_names = list(cfg.cast)
|
| 249 |
obs.log(
|
| 250 |
"registry.cast_assembled",
|
| 251 |
scenario=cfg.name,
|
src/core/run_index.py
CHANGED
|
@@ -56,7 +56,14 @@ class RunSummary(BaseModel):
|
|
| 56 |
finished_at: datetime | None = None
|
| 57 |
reason: str | None = None
|
| 58 |
winner: str | None = None
|
|
|
|
|
|
|
|
|
|
| 59 |
winning_model: str | None = None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
turns: int = 0
|
| 61 |
tokens: int = 0
|
| 62 |
|
|
@@ -88,7 +95,9 @@ def _apply_finished(summary: RunSummary, event: Event) -> None:
|
|
| 88 |
payload = event.payload
|
| 89 |
summary.reason = payload.get("reason")
|
| 90 |
summary.winner = payload.get("winner")
|
|
|
|
| 91 |
summary.winning_model = payload.get("winning_model")
|
|
|
|
| 92 |
summary.turns = int(payload.get("turns", 0) or 0)
|
| 93 |
summary.tokens = int(payload.get("tokens", 0) or 0)
|
| 94 |
summary.finished_at = event.created_at
|
|
|
|
| 56 |
finished_at: datetime | None = None
|
| 57 |
reason: str | None = None
|
| 58 |
winner: str | None = None
|
| 59 |
+
winner_kind: str | None = None
|
| 60 |
+
"""Whether ``winner`` names a cast agent (``"agent"``) or a team label
|
| 61 |
+
(``"team"``); ``None`` for runs with no competition or no winner (ADR-0029)."""
|
| 62 |
winning_model: str | None = None
|
| 63 |
+
winning_models: list[str] = Field(default_factory=list)
|
| 64 |
+
"""Endpoint(s) behind the winner β the agent's model, or every member of a
|
| 65 |
+
winning team (``None`` entries dropped). ``winning_model`` stays the single
|
| 66 |
+
agent-winner endpoint for back-compat."""
|
| 67 |
turns: int = 0
|
| 68 |
tokens: int = 0
|
| 69 |
|
|
|
|
| 95 |
payload = event.payload
|
| 96 |
summary.reason = payload.get("reason")
|
| 97 |
summary.winner = payload.get("winner")
|
| 98 |
+
summary.winner_kind = payload.get("winner_kind")
|
| 99 |
summary.winning_model = payload.get("winning_model")
|
| 100 |
+
summary.winning_models = list(payload.get("winning_models") or [])
|
| 101 |
summary.turns = int(payload.get("turns", 0) or 0)
|
| 102 |
summary.tokens = int(payload.get("tokens", 0) or 0)
|
| 103 |
summary.finished_at = event.created_at
|
src/core/structured.py
CHANGED
|
@@ -40,6 +40,34 @@ class AgentOutputError(ValueError):
|
|
| 40 |
"""Raised when output cannot be normalised to a valid event payload."""
|
| 41 |
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
# ββ validated output model (live path) βββββββββββββββββββββββββββββββββββββββββ
|
| 44 |
|
| 45 |
|
|
@@ -50,22 +78,27 @@ def build_output_model(
|
|
| 50 |
"""Build a Pydantic model for an agent's validated output.
|
| 51 |
|
| 52 |
``kind`` is constrained to *allowed_kinds* via a ``Literal``, so the model
|
| 53 |
-
cannot emit a kind it is not authorised for; ``text``
|
| 54 |
-
are required strings
|
| 55 |
-
|
| 56 |
-
|
|
|
|
|
|
|
|
|
|
| 57 |
|
| 58 |
Args:
|
| 59 |
allowed_kinds: event kinds this agent may emit (the ``may_emit`` grant,
|
| 60 |
reflection excluded). Must be non-empty.
|
| 61 |
-
extra_fields: optional additional payload fields (e.g. ``"emotion"``)
|
| 62 |
-
|
|
|
|
| 63 |
"""
|
| 64 |
if not allowed_kinds:
|
| 65 |
raise AgentOutputError("build_output_model requires at least one allowed kind")
|
| 66 |
|
| 67 |
from pydantic import create_model
|
| 68 |
|
|
|
|
| 69 |
# A single-element Literal is legal and still constrains to that one kind.
|
| 70 |
kind_type = Literal[tuple(allowed_kinds)] # type: ignore[valid-type]
|
| 71 |
fields: dict[str, Any] = {
|
|
@@ -73,7 +106,8 @@ def build_output_model(
|
|
| 73 |
"text": (str, ...),
|
| 74 |
}
|
| 75 |
for name in extra_fields or []:
|
| 76 |
-
|
|
|
|
| 77 |
|
| 78 |
return create_model(
|
| 79 |
"AgentOutput",
|
|
@@ -88,17 +122,40 @@ def build_output_model(
|
|
| 88 |
def json_instruction(allowed_kinds: list[str], extra_fields: list[str] | None = None) -> str:
|
| 89 |
"""Return the JSON constraint block appended to every agent prompt.
|
| 90 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
Args:
|
| 92 |
allowed_kinds: event kinds this agent may emit.
|
| 93 |
extra_fields: optional additional payload fields (e.g. "emotion", "wants").
|
| 94 |
"""
|
| 95 |
-
|
|
|
|
| 96 |
kinds_str = " | ".join(allowed_kinds)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
return (
|
| 98 |
"\n\nOUTPUT FORMAT\n"
|
| 99 |
"Reply with a single JSON object and NOTHING else. No analysis, no reasoning, "
|
| 100 |
"no <think> blocks, no markdown fences, no text before or after the JSON.\n"
|
| 101 |
-
f
|
| 102 |
f"kind must be one of: {kinds_str}\n"
|
| 103 |
"text must be one or two sentences, vivid and specific β your line, never your reasoning.\n"
|
| 104 |
"If you were given a secret word, never spell or quote it; describe it only.\n"
|
|
|
|
| 40 |
"""Raised when output cannot be normalised to a valid event payload."""
|
| 41 |
|
| 42 |
|
| 43 |
+
# ββ well-known typed extra fields ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
# A small, curated table of *engine-known* extra fields (ADR-0029). Most
|
| 47 |
+
# ``output_extra_fields`` are arbitrary scenario fields (``mood``, ``wants``, β¦) and
|
| 48 |
+
# remain required strings, but ``winner`` and ``scores`` are the verdict contract
|
| 49 |
+
# ADR-0026 already names in ``run.finished`` β so the engine gives them real types:
|
| 50 |
+
# an *optional* cast name and an *optional* ``{player: score}`` map. Each entry pairs
|
| 51 |
+
# the Pydantic field spec (built lazily so ``Field`` stays a local import) with the
|
| 52 |
+
# JSON-schema hint ``json_instruction`` renders for that field. Anything not in this
|
| 53 |
+
# table hits the *other* row and behaves exactly as before β back-compat is total.
|
| 54 |
+
def _well_known_specs() -> dict[str, tuple[Any, str]]:
|
| 55 |
+
"""Return ``{field: (pydantic_spec, json_hint)}`` for engine-known extra fields.
|
| 56 |
+
|
| 57 |
+
Built behind a function so ``Field`` is a local import (matching the lazy-import
|
| 58 |
+
idiom of ``build_output_model``) and never touched on the offline path that
|
| 59 |
+
doesn't construct a validated model.
|
| 60 |
+
"""
|
| 61 |
+
from pydantic import Field
|
| 62 |
+
|
| 63 |
+
# The hint is the literal JSON value to render after ``"<field>": `` β a quoted
|
| 64 |
+
# string for ``winner``, a bare object for ``scores`` (it is not a string value).
|
| 65 |
+
return {
|
| 66 |
+
"winner": ((str | None, None), '"<a player\'s name, or null>"'),
|
| 67 |
+
"scores": ((dict[str, float], Field(default_factory=dict)), '{"<player>": 0-10}'),
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
# ββ validated output model (live path) βββββββββββββββββββββββββββββββββββββββββ
|
| 72 |
|
| 73 |
|
|
|
|
| 78 |
"""Build a Pydantic model for an agent's validated output.
|
| 79 |
|
| 80 |
``kind`` is constrained to *allowed_kinds* via a ``Literal``, so the model
|
| 81 |
+
cannot emit a kind it is not authorised for; ``text`` is a required string.
|
| 82 |
+
*extra_fields* are required strings too, *except* the **well-known typed
|
| 83 |
+
fields** of ADR-0029: ``winner`` becomes ``str | None`` (default ``None``) and
|
| 84 |
+
``scores`` becomes ``dict[str, float]`` (default ``{}``). Used on the live path
|
| 85 |
+
with structured output: the provider retries on validation failure and returns a
|
| 86 |
+
valid instance, which means the malformed-prose ``_raw_fallback`` path is never
|
| 87 |
+
taken.
|
| 88 |
|
| 89 |
Args:
|
| 90 |
allowed_kinds: event kinds this agent may emit (the ``may_emit`` grant,
|
| 91 |
reflection excluded). Must be non-empty.
|
| 92 |
+
extra_fields: optional additional payload fields (e.g. ``"emotion"``). Each
|
| 93 |
+
is a required string alongside ``text`` unless it is a well-known typed
|
| 94 |
+
field (``winner``, ``scores``), which is optional with a typed default.
|
| 95 |
"""
|
| 96 |
if not allowed_kinds:
|
| 97 |
raise AgentOutputError("build_output_model requires at least one allowed kind")
|
| 98 |
|
| 99 |
from pydantic import create_model
|
| 100 |
|
| 101 |
+
well_known = _well_known_specs()
|
| 102 |
# A single-element Literal is legal and still constrains to that one kind.
|
| 103 |
kind_type = Literal[tuple(allowed_kinds)] # type: ignore[valid-type]
|
| 104 |
fields: dict[str, Any] = {
|
|
|
|
| 106 |
"text": (str, ...),
|
| 107 |
}
|
| 108 |
for name in extra_fields or []:
|
| 109 |
+
spec, _hint = well_known.get(name, ((str, ...), None))
|
| 110 |
+
fields[name] = spec
|
| 111 |
|
| 112 |
return create_model(
|
| 113 |
"AgentOutput",
|
|
|
|
| 122 |
def json_instruction(allowed_kinds: list[str], extra_fields: list[str] | None = None) -> str:
|
| 123 |
"""Return the JSON constraint block appended to every agent prompt.
|
| 124 |
|
| 125 |
+
For ordinary fields the schema hint is the uniform ``"...": "..."`` shape. When
|
| 126 |
+
a **well-known typed field** of ADR-0029 is present, that field gets a typed hint
|
| 127 |
+
instead (``"winner": "<a player's name, or null>"``,
|
| 128 |
+
``"scores": {"<player>": 0-10}``) so a small model knows it may answer ``null`` or
|
| 129 |
+
a number map rather than a sentence. Manifests with no well-known field render
|
| 130 |
+
byte-identically to the original uniform schema.
|
| 131 |
+
|
| 132 |
Args:
|
| 133 |
allowed_kinds: event kinds this agent may emit.
|
| 134 |
extra_fields: optional additional payload fields (e.g. "emotion", "wants").
|
| 135 |
"""
|
| 136 |
+
fields = extra_fields or []
|
| 137 |
+
well_known = _well_known_specs()
|
| 138 |
kinds_str = " | ".join(allowed_kinds)
|
| 139 |
+
|
| 140 |
+
if any(name in well_known for name in fields):
|
| 141 |
+
# Richer per-field schema: typed hints for known fields, "..." for the rest.
|
| 142 |
+
# Only taken when a well-known field is present, so the common case below
|
| 143 |
+
# stays byte-identical to the original uniform-schema output.
|
| 144 |
+
hints = {"kind": '"..."', "text": '"..."'}
|
| 145 |
+
for name in fields:
|
| 146 |
+
_spec, hint = well_known.get(name, (None, None))
|
| 147 |
+
hints[name] = hint if hint is not None else '"..."'
|
| 148 |
+
schema_body = ", ".join(f'"{name}": {hints[name]}' for name in ["kind", "text", *fields])
|
| 149 |
+
schema_line = f"Schema: {{{schema_body}}}\n"
|
| 150 |
+
else:
|
| 151 |
+
field_list = '", "'.join(["kind", "text"] + list(fields))
|
| 152 |
+
schema_line = f'Schema: {{"{field_list}": "..."}}\n'
|
| 153 |
+
|
| 154 |
return (
|
| 155 |
"\n\nOUTPUT FORMAT\n"
|
| 156 |
"Reply with a single JSON object and NOTHING else. No analysis, no reasoning, "
|
| 157 |
"no <think> blocks, no markdown fences, no text before or after the JSON.\n"
|
| 158 |
+
f"{schema_line}"
|
| 159 |
f"kind must be one of: {kinds_str}\n"
|
| 160 |
"text must be one or two sentences, vivid and specific β your line, never your reasoning.\n"
|
| 161 |
"If you were given a secret word, never spell or quote it; describe it only.\n"
|
src/models/provider.py
CHANGED
|
@@ -4,6 +4,7 @@ import hashlib
|
|
| 4 |
import json
|
| 5 |
import re
|
| 6 |
from dataclasses import dataclass, field
|
|
|
|
| 7 |
|
| 8 |
from src import observability as obs
|
| 9 |
|
|
@@ -256,7 +257,7 @@ class DeterministicTinyModel(ModelProvider):
|
|
| 256 |
allowed_kinds, fields = schema
|
| 257 |
extra = [f for f in fields if f not in ("kind", "text")]
|
| 258 |
if extra: # only agents that opted into extra fields take the JSON path
|
| 259 |
-
obj: dict[str,
|
| 260 |
"kind": allowed_kinds[int(digest[2:4], 16) % len(allowed_kinds)],
|
| 261 |
"text": text,
|
| 262 |
}
|
|
@@ -293,7 +294,7 @@ class DeterministicTinyModel(ModelProvider):
|
|
| 293 |
obs.log("llm.exchange", level="debug", role=role, model=self.variant, prompt=prompt, completion=out)
|
| 294 |
return out
|
| 295 |
|
| 296 |
-
def _synth_field(self, name: str, role: str, digest: str) ->
|
| 297 |
"""Deterministically synthesise a value for one requested extra field."""
|
| 298 |
if name == "mood":
|
| 299 |
moods = _STUB_MOODS_BY_ROLE.get(role, _STUB_MOODS)
|
|
@@ -301,5 +302,13 @@ class DeterministicTinyModel(ModelProvider):
|
|
| 301 |
if name == "thought":
|
| 302 |
opts = _STUB_THOUGHTS.get(role, _STUB_THOUGHT_DEFAULT)
|
| 303 |
return opts[int(digest[6:8], 16) % len(opts)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 304 |
# Unknown extra field: a short, stable placeholder keeps the output valid.
|
| 305 |
return f"{name}:{digest[:4]}"
|
|
|
|
| 4 |
import json
|
| 5 |
import re
|
| 6 |
from dataclasses import dataclass, field
|
| 7 |
+
from typing import Any
|
| 8 |
|
| 9 |
from src import observability as obs
|
| 10 |
|
|
|
|
| 257 |
allowed_kinds, fields = schema
|
| 258 |
extra = [f for f in fields if f not in ("kind", "text")]
|
| 259 |
if extra: # only agents that opted into extra fields take the JSON path
|
| 260 |
+
obj: dict[str, Any] = {
|
| 261 |
"kind": allowed_kinds[int(digest[2:4], 16) % len(allowed_kinds)],
|
| 262 |
"text": text,
|
| 263 |
}
|
|
|
|
| 294 |
obs.log("llm.exchange", level="debug", role=role, model=self.variant, prompt=prompt, completion=out)
|
| 295 |
return out
|
| 296 |
|
| 297 |
+
def _synth_field(self, name: str, role: str, digest: str) -> Any:
|
| 298 |
"""Deterministically synthesise a value for one requested extra field."""
|
| 299 |
if name == "mood":
|
| 300 |
moods = _STUB_MOODS_BY_ROLE.get(role, _STUB_MOODS)
|
|
|
|
| 302 |
if name == "thought":
|
| 303 |
opts = _STUB_THOUGHTS.get(role, _STUB_THOUGHT_DEFAULT)
|
| 304 |
return opts[int(digest[6:8], 16) % len(opts)]
|
| 305 |
+
# Well-known verdict fields (ADR-0029) get their real types, not a placeholder:
|
| 306 |
+
# the stub names no winner (the field is optional, and a versus handler recovers
|
| 307 |
+
# the accusation from the verdict text), and emits an empty score map β so the
|
| 308 |
+
# offline path stays validation-clean and deterministic with no wasted re-ask.
|
| 309 |
+
if name == "winner":
|
| 310 |
+
return None
|
| 311 |
+
if name == "scores":
|
| 312 |
+
return {}
|
| 313 |
# Unknown extra field: a short, stable placeholder keeps the output valid.
|
| 314 |
return f"{name}:{digest[:4]}"
|
src/ui/fishbowl/session.py
CHANGED
|
@@ -115,11 +115,15 @@ class FishbowlSession:
|
|
| 115 |
def finalize(self, reason: str) -> None:
|
| 116 |
"""Close the current run with a ``run.finished`` event (idempotent-safe).
|
| 117 |
|
| 118 |
-
On a verdict we derive ``winner`` from the judge's ruling
|
| 119 |
-
|
| 120 |
-
|
|
|
|
|
|
|
| 121 |
winner: str | None = None
|
|
|
|
| 122 |
winning_model: str | None = None
|
|
|
|
| 123 |
run_events = self.conductor.ledger.events_for_run(self.conductor.run_id)
|
| 124 |
if reason == "verdict":
|
| 125 |
verdict = next((e for e in reversed(run_events) if e.kind == "judge.verdict"), None)
|
|
@@ -128,8 +132,26 @@ class FishbowlSession:
|
|
| 128 |
if winner:
|
| 129 |
started = next((e for e in run_events if e.kind == "run.started"), None)
|
| 130 |
cast = (started.payload.get("cast") or {}) if started is not None else {}
|
| 131 |
-
|
| 132 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
|
| 134 |
@property
|
| 135 |
def cast(self) -> list[AgentManifest]:
|
|
|
|
| 115 |
def finalize(self, reason: str) -> None:
|
| 116 |
"""Close the current run with a ``run.finished`` event (idempotent-safe).
|
| 117 |
|
| 118 |
+
On a verdict we derive ``winner`` from the judge's ruling and resolve its kind
|
| 119 |
+
(ADR-0029): a cast agent name β ``winner_kind: "agent"`` with that agent's
|
| 120 |
+
``winning_model``; a team label β ``winner_kind: "team"`` with every member's
|
| 121 |
+
endpoint in ``winning_models`` (``winning_model`` left ``None``, never guessed).
|
| 122 |
+
All fall back to ``None`` / empty when unknown."""
|
| 123 |
winner: str | None = None
|
| 124 |
+
winner_kind: str | None = None
|
| 125 |
winning_model: str | None = None
|
| 126 |
+
winning_models: list[str] = []
|
| 127 |
run_events = self.conductor.ledger.events_for_run(self.conductor.run_id)
|
| 128 |
if reason == "verdict":
|
| 129 |
verdict = next((e for e in reversed(run_events) if e.kind == "judge.verdict"), None)
|
|
|
|
| 132 |
if winner:
|
| 133 |
started = next((e for e in run_events if e.kind == "run.started"), None)
|
| 134 |
cast = (started.payload.get("cast") or {}) if started is not None else {}
|
| 135 |
+
scenario = self._registry.scenarios.get(self._scenario_name)
|
| 136 |
+
teams = getattr(getattr(scenario, "competition", None), "teams", None) or {}
|
| 137 |
+
if winner in cast:
|
| 138 |
+
winner_kind = "agent"
|
| 139 |
+
winning_model = (cast.get(winner) or {}).get("model_endpoint")
|
| 140 |
+
winning_models = [winning_model] if winning_model else []
|
| 141 |
+
elif winner in teams:
|
| 142 |
+
winner_kind = "team"
|
| 143 |
+
winning_models = [
|
| 144 |
+
endpoint
|
| 145 |
+
for member in teams[winner]
|
| 146 |
+
if (endpoint := (cast.get(member) or {}).get("model_endpoint"))
|
| 147 |
+
]
|
| 148 |
+
self.conductor.finalize(
|
| 149 |
+
reason,
|
| 150 |
+
winner=winner,
|
| 151 |
+
winner_kind=winner_kind,
|
| 152 |
+
winning_model=winning_model,
|
| 153 |
+
winning_models=winning_models,
|
| 154 |
+
)
|
| 155 |
|
| 156 |
@property
|
| 157 |
def cast(self) -> list[AgentManifest]:
|
tests/test_config.py
CHANGED
|
@@ -4,6 +4,7 @@ import pytest
|
|
| 4 |
from pydantic import ValidationError
|
| 5 |
|
| 6 |
from src.core.config import (
|
|
|
|
| 7 |
ModelProfileConfig,
|
| 8 |
ModelsConfig,
|
| 9 |
ScenarioConfig,
|
|
@@ -36,9 +37,7 @@ class TestValidateAgent:
|
|
| 36 |
|
| 37 |
class TestValidateScenario:
|
| 38 |
def test_valid_with_goal_and_cast(self):
|
| 39 |
-
s = validate_scenario(
|
| 40 |
-
{"name": "w", "goal": "be strange", "default_seed": "seed", "cast": ["a", "b"]}
|
| 41 |
-
)
|
| 42 |
assert s.goal == "be strange"
|
| 43 |
assert s.cast == ["a", "b"]
|
| 44 |
|
|
@@ -65,3 +64,131 @@ class TestValidateWorld:
|
|
| 65 |
}
|
| 66 |
)
|
| 67 |
assert "undefined agents" in str(exc.value)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
from pydantic import ValidationError
|
| 5 |
|
| 6 |
from src.core.config import (
|
| 7 |
+
CompetitionConfig,
|
| 8 |
ModelProfileConfig,
|
| 9 |
ModelsConfig,
|
| 10 |
ScenarioConfig,
|
|
|
|
| 37 |
|
| 38 |
class TestValidateScenario:
|
| 39 |
def test_valid_with_goal_and_cast(self):
|
| 40 |
+
s = validate_scenario({"name": "w", "goal": "be strange", "default_seed": "seed", "cast": ["a", "b"]})
|
|
|
|
|
|
|
| 41 |
assert s.goal == "be strange"
|
| 42 |
assert s.cast == ["a", "b"]
|
| 43 |
|
|
|
|
| 64 |
}
|
| 65 |
)
|
| 66 |
assert "undefined agents" in str(exc.value)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
# ββ competition contract (ADR-0029) ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
class TestCompetitionConfig:
|
| 73 |
+
"""A scenario's winner contract β versus/judged/none and the team-shape rules
|
| 74 |
+
a competition can enforce on its own (cross-cast checks live on WorldConfig)."""
|
| 75 |
+
|
| 76 |
+
def test_default_is_none_with_no_teams(self):
|
| 77 |
+
# An absent block == none; the field must default safely, never to versus.
|
| 78 |
+
c = CompetitionConfig()
|
| 79 |
+
assert c.kind == "none"
|
| 80 |
+
assert c.teams is None
|
| 81 |
+
|
| 82 |
+
def test_judged_needs_no_teams(self):
|
| 83 |
+
c = CompetitionConfig(kind="judged")
|
| 84 |
+
assert c.kind == "judged"
|
| 85 |
+
assert c.teams is None
|
| 86 |
+
|
| 87 |
+
@pytest.mark.parametrize("kind", ["none", "judged"])
|
| 88 |
+
def test_teams_forbidden_unless_versus(self, kind):
|
| 89 |
+
# teams on a non-versus kind is a config mistake β the winner has no team map.
|
| 90 |
+
with pytest.raises(ValidationError) as exc:
|
| 91 |
+
CompetitionConfig(kind=kind, teams={"spy": ["a"]})
|
| 92 |
+
assert "only allowed when kind is 'versus'" in str(exc.value)
|
| 93 |
+
|
| 94 |
+
def test_versus_requires_non_empty_teams(self):
|
| 95 |
+
with pytest.raises(ValidationError) as exc:
|
| 96 |
+
CompetitionConfig(kind="versus", teams={})
|
| 97 |
+
assert "non-empty 'teams'" in str(exc.value)
|
| 98 |
+
|
| 99 |
+
def test_versus_with_missing_teams_rejected(self):
|
| 100 |
+
# kind=versus with no teams at all is the same defect as an empty mapping.
|
| 101 |
+
with pytest.raises(ValidationError):
|
| 102 |
+
CompetitionConfig(kind="versus")
|
| 103 |
+
|
| 104 |
+
def test_versus_rejects_empty_member_list(self):
|
| 105 |
+
# A team with no members can never win or lose β reject it at config time.
|
| 106 |
+
with pytest.raises(ValidationError) as exc:
|
| 107 |
+
CompetitionConfig(kind="versus", teams={"spy": ["spy-nil"], "herd": []})
|
| 108 |
+
assert "empty member lists" in str(exc.value)
|
| 109 |
+
assert "herd" in str(exc.value)
|
| 110 |
+
|
| 111 |
+
def test_versus_rejects_overlapping_teams(self):
|
| 112 |
+
# An agent on two teams makes "who won" ambiguous β disjointness is required.
|
| 113 |
+
with pytest.raises(ValidationError) as exc:
|
| 114 |
+
CompetitionConfig(kind="versus", teams={"spy": ["nil"], "herd": ["cara", "nil"]})
|
| 115 |
+
assert "mutually disjoint" in str(exc.value)
|
| 116 |
+
assert "nil" in str(exc.value)
|
| 117 |
+
|
| 118 |
+
def test_versus_disjoint_teams_accepted(self):
|
| 119 |
+
c = CompetitionConfig(kind="versus", teams={"spy": ["nil"], "herd": ["cara", "bex"]})
|
| 120 |
+
assert c.teams == {"spy": ["nil"], "herd": ["cara", "bex"]}
|
| 121 |
+
|
| 122 |
+
def test_same_member_repeated_within_one_team_is_not_overlap(self):
|
| 123 |
+
# Overlap means across DIFFERENT labels; a dup inside one team is harmless here.
|
| 124 |
+
c = CompetitionConfig(kind="versus", teams={"spy": ["nil", "nil"]})
|
| 125 |
+
assert c.kind == "versus"
|
| 126 |
+
|
| 127 |
+
def test_extra_field_forbidden(self):
|
| 128 |
+
with pytest.raises(ValidationError):
|
| 129 |
+
CompetitionConfig(kind="none", bogus=1) # type: ignore[call-arg]
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class TestScenarioCompetition:
|
| 133 |
+
def test_scenario_accepts_competition_block(self):
|
| 134 |
+
s = validate_scenario(
|
| 135 |
+
{
|
| 136 |
+
"name": "duel",
|
| 137 |
+
"default_seed": "seed",
|
| 138 |
+
"cast": ["a", "b"],
|
| 139 |
+
"competition": {"kind": "versus", "teams": {"x": ["a"], "y": ["b"]}},
|
| 140 |
+
}
|
| 141 |
+
)
|
| 142 |
+
assert s.competition is not None
|
| 143 |
+
assert s.competition.kind == "versus"
|
| 144 |
+
|
| 145 |
+
def test_scenario_without_competition_defaults_to_none_attribute(self):
|
| 146 |
+
# Absent block == no competition object (the hook reads this as "none").
|
| 147 |
+
s = validate_scenario({"name": "s", "default_seed": "seed", "cast": ["a"]})
|
| 148 |
+
assert s.competition is None
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
class TestWorldCompetitionCrossChecks:
|
| 152 |
+
"""WorldConfig is where a team's members are checked against the scenario cast
|
| 153 |
+
and team labels against agent names β the rules that need the whole world."""
|
| 154 |
+
|
| 155 |
+
def _world(self, competition: dict) -> dict:
|
| 156 |
+
return {
|
| 157 |
+
"agents": [
|
| 158 |
+
{"name": "spy-nil", "persona": "p"},
|
| 159 |
+
{"name": "spy-cara", "persona": "p"},
|
| 160 |
+
{"name": "host", "persona": "p"},
|
| 161 |
+
],
|
| 162 |
+
"scenarios": [
|
| 163 |
+
{
|
| 164 |
+
"name": "duel",
|
| 165 |
+
"default_seed": "seed",
|
| 166 |
+
"cast": ["spy-nil", "spy-cara", "host"],
|
| 167 |
+
"competition": competition,
|
| 168 |
+
}
|
| 169 |
+
],
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
def test_coherent_versus_world_validates(self):
|
| 173 |
+
world = validate_world(self._world({"kind": "versus", "teams": {"spy": ["spy-nil"], "herd": ["spy-cara"]}}))
|
| 174 |
+
assert world.scenarios[0].competition.kind == "versus"
|
| 175 |
+
|
| 176 |
+
def test_off_cast_team_member_rejected(self):
|
| 177 |
+
# A team naming an agent not in this scenario's cast can never be scored.
|
| 178 |
+
with pytest.raises(ValidationError) as exc:
|
| 179 |
+
validate_world(self._world({"kind": "versus", "teams": {"spy": ["ghost-agent"], "herd": ["spy-cara"]}}))
|
| 180 |
+
assert "team members not in its cast" in str(exc.value)
|
| 181 |
+
assert "ghost-agent" in str(exc.value)
|
| 182 |
+
|
| 183 |
+
def test_team_label_colliding_with_agent_name_rejected(self):
|
| 184 |
+
# winner carries an agent name OR a team label; a label that IS an agent name
|
| 185 |
+
# makes that union ambiguous, so the cross-cast check must reject it.
|
| 186 |
+
with pytest.raises(ValidationError) as exc:
|
| 187 |
+
validate_world(self._world({"kind": "versus", "teams": {"host": ["spy-nil"], "herd": ["spy-cara"]}}))
|
| 188 |
+
assert "collide with agent names" in str(exc.value)
|
| 189 |
+
assert "host" in str(exc.value)
|
| 190 |
+
|
| 191 |
+
def test_judged_scenario_skips_team_checks(self):
|
| 192 |
+
# No teams means the cross-cast loop has nothing to enforce β it must pass.
|
| 193 |
+
world = validate_world(self._world({"kind": "judged"}))
|
| 194 |
+
assert world.scenarios[0].competition.kind == "judged"
|
tests/test_run_history.py
CHANGED
|
@@ -130,6 +130,28 @@ class TestRunFinished:
|
|
| 130 |
assert len(finished) == 1
|
| 131 |
assert finished[0].payload["reason"] == "user_stop"
|
| 132 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
def test_session_finalize_derives_winner_and_model_from_verdict(self):
|
| 134 |
scenario = _verdict_scenario()
|
| 135 |
# Build a session-like conductor manually so we control the cast verdict.
|
|
@@ -302,7 +324,9 @@ def _bookended_runs() -> list[Event]:
|
|
| 302 |
payload={
|
| 303 |
"reason": "verdict",
|
| 304 |
"winner": "judge",
|
|
|
|
| 305 |
"winning_model": "model://j",
|
|
|
|
| 306 |
"turns": 2,
|
| 307 |
"tokens": 1234,
|
| 308 |
},
|
|
@@ -312,6 +336,42 @@ def _bookended_runs() -> list[Event]:
|
|
| 312 |
]
|
| 313 |
|
| 314 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
class TestRunIndexModule:
|
| 316 |
def test_index_runs_folds_bookend_events(self):
|
| 317 |
summaries = index_runs(_bookended_runs())
|
|
@@ -323,6 +383,9 @@ class TestRunIndexModule:
|
|
| 323 |
assert r1.cast["judge"].model_endpoint == "model://j"
|
| 324 |
assert r1.cast["judge"].model_profile == "strong"
|
| 325 |
assert (r1.reason, r1.winner, r1.winning_model) == ("verdict", "judge", "model://j")
|
|
|
|
|
|
|
|
|
|
| 326 |
assert (r1.turns, r1.tokens) == (2, 1234)
|
| 327 |
assert r1.started_at is not None and r1.finished_at is not None
|
| 328 |
|
|
@@ -331,12 +394,26 @@ class TestRunIndexModule:
|
|
| 331 |
assert r2.reason is None and r2.winner is None
|
| 332 |
assert (r2.turns, r2.tokens) == (0, 0)
|
| 333 |
assert r2.finished_at is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 334 |
|
| 335 |
@pytest.mark.parametrize("make_ledger", [Ledger, lambda: SqlAlchemyLedger("sqlite://")])
|
| 336 |
-
|
| 337 |
-
|
|
|
|
| 338 |
ledger = make_ledger()
|
| 339 |
for e in events:
|
| 340 |
ledger.append(e)
|
| 341 |
-
# The indexed-query path must produce exactly what the pure oracle does
|
|
|
|
| 342 |
assert index_runs_from_ledger(ledger) == index_runs(events)
|
|
|
|
| 130 |
assert len(finished) == 1
|
| 131 |
assert finished[0].payload["reason"] == "user_stop"
|
| 132 |
|
| 133 |
+
def test_versus_session_finalizes_with_team_winner(self):
|
| 134 |
+
# End-to-end offline: the SpyHost stamps a team winner ("herd"/"spy") on the
|
| 135 |
+
# verdict, and FishbowlSession.finalize must resolve winner_kind == "team"
|
| 136 |
+
# (a team label, not a cast agent), never guessing a single winning_model.
|
| 137 |
+
session = FishbowlSession("the-steeped")
|
| 138 |
+
session.reset()
|
| 139 |
+
for _ in range(session.autoplay_tick_cap):
|
| 140 |
+
if session.has_verdict():
|
| 141 |
+
break
|
| 142 |
+
session.step()
|
| 143 |
+
assert session.has_verdict(), "the stub spy-host should reach a verdict"
|
| 144 |
+
|
| 145 |
+
session.finalize("verdict")
|
| 146 |
+
finished = [
|
| 147 |
+
e for e in session.conductor.ledger.events_for_run(session.conductor.run_id) if e.kind == "run.finished"
|
| 148 |
+
]
|
| 149 |
+
assert len(finished) == 1
|
| 150 |
+
payload = finished[0].payload
|
| 151 |
+
assert payload["winner"] in ("herd", "spy") # a team label, code-stamped
|
| 152 |
+
assert payload["winner_kind"] == "team"
|
| 153 |
+
assert payload["winning_model"] is None # never guessed for a team win
|
| 154 |
+
|
| 155 |
def test_session_finalize_derives_winner_and_model_from_verdict(self):
|
| 156 |
scenario = _verdict_scenario()
|
| 157 |
# Build a session-like conductor manually so we control the cast verdict.
|
|
|
|
| 324 |
payload={
|
| 325 |
"reason": "verdict",
|
| 326 |
"winner": "judge",
|
| 327 |
+
"winner_kind": "agent",
|
| 328 |
"winning_model": "model://j",
|
| 329 |
+
"winning_models": ["model://j"],
|
| 330 |
"turns": 2,
|
| 331 |
"tokens": 1234,
|
| 332 |
},
|
|
|
|
| 336 |
]
|
| 337 |
|
| 338 |
|
| 339 |
+
def _team_win_run() -> list[Event]:
|
| 340 |
+
"""A single versus run finishing on a TEAM win (winner_kind 'team', no single model)."""
|
| 341 |
+
return [
|
| 342 |
+
Event(
|
| 343 |
+
run_id="v1",
|
| 344 |
+
turn=0,
|
| 345 |
+
kind="run.started",
|
| 346 |
+
actor="conductor",
|
| 347 |
+
payload={
|
| 348 |
+
"seed": "leaf",
|
| 349 |
+
"scenario": "the-steeped",
|
| 350 |
+
"cast": {
|
| 351 |
+
"spy-cara": {"model_endpoint": "model://cara", "model_profile": "fast"},
|
| 352 |
+
"spy-bex": {"model_endpoint": "model://bex", "model_profile": "fast"},
|
| 353 |
+
"spy-ovo": {"model_endpoint": None, "model_profile": "fast"},
|
| 354 |
+
},
|
| 355 |
+
},
|
| 356 |
+
),
|
| 357 |
+
Event(
|
| 358 |
+
run_id="v1",
|
| 359 |
+
turn=2,
|
| 360 |
+
kind="run.finished",
|
| 361 |
+
actor="conductor",
|
| 362 |
+
payload={
|
| 363 |
+
"reason": "verdict",
|
| 364 |
+
"winner": "herd",
|
| 365 |
+
"winner_kind": "team",
|
| 366 |
+
"winning_model": None, # never guessed for a team win
|
| 367 |
+
"winning_models": ["model://cara", "model://bex"], # None member endpoint dropped
|
| 368 |
+
"turns": 2,
|
| 369 |
+
"tokens": 50,
|
| 370 |
+
},
|
| 371 |
+
),
|
| 372 |
+
]
|
| 373 |
+
|
| 374 |
+
|
| 375 |
class TestRunIndexModule:
|
| 376 |
def test_index_runs_folds_bookend_events(self):
|
| 377 |
summaries = index_runs(_bookended_runs())
|
|
|
|
| 383 |
assert r1.cast["judge"].model_endpoint == "model://j"
|
| 384 |
assert r1.cast["judge"].model_profile == "strong"
|
| 385 |
assert (r1.reason, r1.winner, r1.winning_model) == ("verdict", "judge", "model://j")
|
| 386 |
+
# ADR-0029 attribution keys round-trip through the projection (agent win).
|
| 387 |
+
assert r1.winner_kind == "agent"
|
| 388 |
+
assert r1.winning_models == ["model://j"]
|
| 389 |
assert (r1.turns, r1.tokens) == (2, 1234)
|
| 390 |
assert r1.started_at is not None and r1.finished_at is not None
|
| 391 |
|
|
|
|
| 394 |
assert r2.reason is None and r2.winner is None
|
| 395 |
assert (r2.turns, r2.tokens) == (0, 0)
|
| 396 |
assert r2.finished_at is None
|
| 397 |
+
# An unfinished run carries the additive ADR-0029 defaults, never a stale guess.
|
| 398 |
+
assert r2.winner_kind is None
|
| 399 |
+
assert r2.winning_models == []
|
| 400 |
+
|
| 401 |
+
def test_index_runs_round_trips_a_team_win(self):
|
| 402 |
+
# winner_kind 'team' carries no single winning_model, only the member endpoints β
|
| 403 |
+
# and a None member endpoint is dropped from winning_models.
|
| 404 |
+
(summary,) = index_runs(_team_win_run())
|
| 405 |
+
assert summary.winner == "herd"
|
| 406 |
+
assert summary.winner_kind == "team"
|
| 407 |
+
assert summary.winning_model is None
|
| 408 |
+
assert summary.winning_models == ["model://cara", "model://bex"]
|
| 409 |
|
| 410 |
@pytest.mark.parametrize("make_ledger", [Ledger, lambda: SqlAlchemyLedger("sqlite://")])
|
| 411 |
+
@pytest.mark.parametrize("events_factory", [_bookended_runs, _team_win_run])
|
| 412 |
+
def test_from_ledger_matches_pure_oracle(self, make_ledger, events_factory):
|
| 413 |
+
events = events_factory()
|
| 414 |
ledger = make_ledger()
|
| 415 |
for e in events:
|
| 416 |
ledger.append(e)
|
| 417 |
+
# The indexed-query path must produce exactly what the pure oracle does β
|
| 418 |
+
# including the additive ADR-0029 attribution keys (agent win and team win).
|
| 419 |
assert index_runs_from_ledger(ledger) == index_runs(events)
|
tests/test_structured.py
CHANGED
|
@@ -1,6 +1,11 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
|
|
|
|
|
|
| 3 |
from src.core.structured import (
|
|
|
|
|
|
|
| 4 |
clean_clue,
|
| 5 |
extract_reasoning,
|
| 6 |
is_usable_line,
|
|
@@ -198,3 +203,85 @@ class TestIsUsableLine:
|
|
| 198 |
|
| 199 |
def test_accepts_a_real_line(self):
|
| 200 |
assert is_usable_line("A dark brew warms the dawn.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
import pytest
|
| 4 |
+
from pydantic import ValidationError
|
| 5 |
+
|
| 6 |
from src.core.structured import (
|
| 7 |
+
AgentOutputError,
|
| 8 |
+
build_output_model,
|
| 9 |
clean_clue,
|
| 10 |
extract_reasoning,
|
| 11 |
is_usable_line,
|
|
|
|
| 203 |
|
| 204 |
def test_accepts_a_real_line(self):
|
| 205 |
assert is_usable_line("A dark brew warms the dawn.")
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
class TestBuildOutputModel:
|
| 209 |
+
"""The live-path schema. kind is Literal-constrained, text is required, and the
|
| 210 |
+
well-known verdict fields (ADR-0029) get real optional types β everything else
|
| 211 |
+
stays a required string, exactly as before."""
|
| 212 |
+
|
| 213 |
+
def test_requires_at_least_one_kind(self):
|
| 214 |
+
with pytest.raises(AgentOutputError):
|
| 215 |
+
build_output_model([])
|
| 216 |
+
|
| 217 |
+
def test_kind_is_literal_constrained(self):
|
| 218 |
+
model = build_output_model(["judge.verdict"])
|
| 219 |
+
with pytest.raises(ValidationError):
|
| 220 |
+
model(kind="not.allowed", text="x")
|
| 221 |
+
|
| 222 |
+
def test_text_is_required(self):
|
| 223 |
+
model = build_output_model(["agent.spoke"])
|
| 224 |
+
with pytest.raises(ValidationError):
|
| 225 |
+
model(kind="agent.spoke")
|
| 226 |
+
|
| 227 |
+
def test_ordinary_extra_field_is_required_string(self):
|
| 228 |
+
# The *other* row: an arbitrary scenario field stays a required str (back-compat).
|
| 229 |
+
model = build_output_model(["agent.spoke"], extra_fields=["mood"])
|
| 230 |
+
with pytest.raises(ValidationError):
|
| 231 |
+
model(kind="agent.spoke", text="hi") # mood missing β invalid
|
| 232 |
+
inst = model(kind="agent.spoke", text="hi", mood="calm")
|
| 233 |
+
assert inst.mood == "calm"
|
| 234 |
+
|
| 235 |
+
def test_winner_is_optional_and_defaults_to_none(self):
|
| 236 |
+
# A judge may decline to name a winner; the field must default to None, not error.
|
| 237 |
+
model = build_output_model(["judge.verdict"], extra_fields=["winner"])
|
| 238 |
+
inst = model(kind="judge.verdict", text="Verdict: undecided.")
|
| 239 |
+
assert inst.winner is None
|
| 240 |
+
|
| 241 |
+
def test_winner_accepts_a_name(self):
|
| 242 |
+
model = build_output_model(["judge.verdict"], extra_fields=["winner"])
|
| 243 |
+
inst = model(kind="judge.verdict", text="t", winner="clue-gatherer")
|
| 244 |
+
assert inst.winner == "clue-gatherer"
|
| 245 |
+
|
| 246 |
+
def test_scores_defaults_to_empty_map(self):
|
| 247 |
+
model = build_output_model(["judge.verdict"], extra_fields=["scores"])
|
| 248 |
+
inst = model(kind="judge.verdict", text="t")
|
| 249 |
+
assert inst.scores == {}
|
| 250 |
+
|
| 251 |
+
def test_scores_coerces_numeric_map(self):
|
| 252 |
+
model = build_output_model(["judge.verdict"], extra_fields=["scores"])
|
| 253 |
+
inst = model(kind="judge.verdict", text="t", scores={"clue-gatherer": 9})
|
| 254 |
+
assert inst.scores == {"clue-gatherer": 9.0}
|
| 255 |
+
|
| 256 |
+
def test_mixed_known_and_unknown_fields(self):
|
| 257 |
+
# mood required, winner/scores optional β the full mystery-judge shape.
|
| 258 |
+
model = build_output_model(["judge.verdict"], extra_fields=["mood", "winner", "scores"])
|
| 259 |
+
inst = model(kind="judge.verdict", text="t", mood="smug")
|
| 260 |
+
assert (inst.mood, inst.winner, inst.scores) == ("smug", None, {})
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
class TestJsonInstructionWellKnown:
|
| 264 |
+
"""The prompt hint. With NO well-known field present it must be byte-identical to
|
| 265 |
+
the original uniform schema; with winner/scores present it renders typed hints."""
|
| 266 |
+
|
| 267 |
+
@pytest.mark.parametrize("extra", [None, ["mood"], ["thought"], ["mood", "thought"]])
|
| 268 |
+
def test_byte_identical_without_well_known_fields(self, extra):
|
| 269 |
+
# The common case must not drift: the same schema-line rendering as before.
|
| 270 |
+
out = json_instruction(["agent.spoke"], extra_fields=extra)
|
| 271 |
+
fields = '", "'.join(["kind", "text", *(extra or [])])
|
| 272 |
+
assert f'Schema: {{"{fields}": "..."}}' in out
|
| 273 |
+
|
| 274 |
+
def test_winner_hint_appears_when_present(self):
|
| 275 |
+
out = json_instruction(["judge.verdict"], extra_fields=["winner"])
|
| 276 |
+
assert "winner" in out
|
| 277 |
+
assert "or null" in out # the typed hint, not the uniform "..."
|
| 278 |
+
|
| 279 |
+
def test_scores_hint_appears_when_present(self):
|
| 280 |
+
out = json_instruction(["judge.verdict"], extra_fields=["scores"])
|
| 281 |
+
assert "0-10" in out # a number-map hint, not a quoted string
|
| 282 |
+
|
| 283 |
+
def test_ordinary_field_keeps_uniform_hint_alongside_known(self):
|
| 284 |
+
# mood sits next to winner: it still gets "..." while winner gets its typed hint.
|
| 285 |
+
out = json_instruction(["judge.verdict"], extra_fields=["mood", "winner", "scores"])
|
| 286 |
+
assert '"mood": "..."' in out
|
| 287 |
+
assert "or null" in out and "0-10" in out
|
tests/test_verdict_validation.py
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Verdict winner validation and the ground-truth scoreboard (ADR-0029).
|
| 2 |
+
|
| 3 |
+
The base agent validates a judge's ``winner`` (re-asking once on a bad pick, summing
|
| 4 |
+
usage, stamping ``no_contest`` on a second failure) and normalises ``scores`` in place.
|
| 5 |
+
The ``SpyHost`` handler then turns the judge's *accusation* into a code-stamped
|
| 6 |
+
*result* using the scenario's ``competition.teams``.
|
| 7 |
+
|
| 8 |
+
Zero mocks: agents are built through the real registry, and the live ``complete_structured``
|
| 9 |
+
seam is exercised with a small hand-written ``FakeProvider`` (the canonical zero-mock
|
| 10 |
+
seam) and an offline ``DeterministicTinyModel`` for the stub path.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
from src.agents.handlers import SpyHost
|
| 16 |
+
from src.core.events import Event
|
| 17 |
+
from src.core.projections import StageProjection
|
| 18 |
+
from src.core.registry import default_registry
|
| 19 |
+
from src.models.router import ModelRouter
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# ββ live-path provider seam (no unittest.mock) ββββββββββββββββββββββββββββββββββββ
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class _ScriptedJudge:
|
| 26 |
+
"""A live provider whose ``complete_structured`` returns scripted verdicts.
|
| 27 |
+
|
| 28 |
+
Exposing ``complete_structured`` is what routes the base agent down the live
|
| 29 |
+
path (``hasattr(provider, "complete_structured")``). Each call returns the next
|
| 30 |
+
scripted ``(winner, scores)`` and reports the matching usage, so a re-ask is a
|
| 31 |
+
real second round-trip the test can count and meter."""
|
| 32 |
+
|
| 33 |
+
def __init__(self, scripts: list[dict]) -> None:
|
| 34 |
+
self._scripts = scripts
|
| 35 |
+
self.calls = 0
|
| 36 |
+
self.last_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
| 37 |
+
self.last_reasoning = ""
|
| 38 |
+
self.model_id = "scripted"
|
| 39 |
+
|
| 40 |
+
def complete_structured(self, role, prompt, model):
|
| 41 |
+
script = self._scripts[min(self.calls, len(self._scripts) - 1)]
|
| 42 |
+
self.calls += 1
|
| 43 |
+
usage = script.get("usage", {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110})
|
| 44 |
+
self.last_usage = dict(usage)
|
| 45 |
+
return model(
|
| 46 |
+
kind=script.get("kind", "judge.verdict"),
|
| 47 |
+
text=script.get("text", "Verdict: someone did it."),
|
| 48 |
+
mood=script.get("mood", "calm"),
|
| 49 |
+
winner=script.get("winner"),
|
| 50 |
+
scores=script.get("scores", {}),
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class _ScriptedRouter(ModelRouter):
|
| 55 |
+
"""A router that always hands back one scripted provider (the live seam)."""
|
| 56 |
+
|
| 57 |
+
def __init__(self, provider) -> None:
|
| 58 |
+
self._provider = provider
|
| 59 |
+
self.offline = False
|
| 60 |
+
|
| 61 |
+
def for_profile(self, key): # type: ignore[override]
|
| 62 |
+
return self._provider
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _judge(scripts: list[dict], *, scenario: str = "mystery-roots", name: str = "mystery-judge"):
|
| 66 |
+
"""Build a judge through the registry, wired to a scripted live provider."""
|
| 67 |
+
reg = default_registry()
|
| 68 |
+
provider = _ScriptedJudge(scripts)
|
| 69 |
+
agent = reg.build_agent(name, _ScriptedRouter(provider))
|
| 70 |
+
cfg = reg.scenarios[scenario]
|
| 71 |
+
agent.competition = cfg.competition
|
| 72 |
+
agent.cast_names = list(cfg.cast)
|
| 73 |
+
return agent, provider
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _resolve(agent):
|
| 77 |
+
return agent._resolve_payload(
|
| 78 |
+
agent.manifest.name,
|
| 79 |
+
"PROMPT",
|
| 80 |
+
agent._content_kinds(),
|
| 81 |
+
agent.manifest.output_extra_fields,
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
# ββ live re-ask flow βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class TestVerdictReask:
|
| 89 |
+
def test_off_cast_winner_triggers_exactly_one_reask(self):
|
| 90 |
+
# The reference flow: bad winner first, valid winner on the corrective re-ask.
|
| 91 |
+
agent, provider = _judge(
|
| 92 |
+
[
|
| 93 |
+
{
|
| 94 |
+
"text": "Verdict: the butler did it.",
|
| 95 |
+
"winner": "NOT-A-CAST-NAME",
|
| 96 |
+
"scores": {"clue-gatherer": 12},
|
| 97 |
+
"usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110},
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"text": "Verdict: the gatherer cracked it.",
|
| 101 |
+
"winner": "clue-gatherer",
|
| 102 |
+
"scores": {"clue-gatherer": 9},
|
| 103 |
+
"usage": {"prompt_tokens": 120, "completion_tokens": 12, "total_tokens": 132},
|
| 104 |
+
},
|
| 105 |
+
]
|
| 106 |
+
)
|
| 107 |
+
payload = _resolve(agent)
|
| 108 |
+
assert provider.calls == 2 # one re-ask, not two, not zero
|
| 109 |
+
assert payload["winner"] == "clue-gatherer"
|
| 110 |
+
assert "no_contest" not in payload
|
| 111 |
+
|
| 112 |
+
def test_reask_usage_is_summed_not_overwritten(self):
|
| 113 |
+
# ADR-0029 acceptance criterion: the governor must meter BOTH calls.
|
| 114 |
+
agent, _ = _judge(
|
| 115 |
+
[
|
| 116 |
+
{"winner": "bad", "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}},
|
| 117 |
+
{
|
| 118 |
+
"winner": "clue-gatherer",
|
| 119 |
+
"text": "Verdict: the gatherer cracked it.",
|
| 120 |
+
"usage": {"prompt_tokens": 120, "completion_tokens": 12, "total_tokens": 132},
|
| 121 |
+
},
|
| 122 |
+
]
|
| 123 |
+
)
|
| 124 |
+
_resolve(agent)
|
| 125 |
+
assert agent.last_usage["total_tokens"] == 242 # 110 + 132
|
| 126 |
+
assert agent.last_usage["prompt_tokens"] == 220
|
| 127 |
+
assert agent.last_usage["completion_tokens"] == 22
|
| 128 |
+
|
| 129 |
+
def test_valid_first_try_does_not_reask(self):
|
| 130 |
+
agent, provider = _judge([{"winner": "hypothesis-former", "text": "Verdict: the hypothesis held."}])
|
| 131 |
+
payload = _resolve(agent)
|
| 132 |
+
assert provider.calls == 1
|
| 133 |
+
assert payload["winner"] == "hypothesis-former"
|
| 134 |
+
assert "no_contest" not in payload
|
| 135 |
+
|
| 136 |
+
def test_reask_that_also_fails_drops_winner_and_stamps_no_contest(self):
|
| 137 |
+
# Two bad picks in a row: the verdict TEXT still ships, the row is forfeited.
|
| 138 |
+
agent, provider = _judge(
|
| 139 |
+
[
|
| 140 |
+
{"winner": "bad-one", "text": "Verdict: I accuse the wind."},
|
| 141 |
+
{"winner": "bad-two", "text": "Verdict: no, the rain."},
|
| 142 |
+
]
|
| 143 |
+
)
|
| 144 |
+
payload = _resolve(agent)
|
| 145 |
+
assert provider.calls == 2
|
| 146 |
+
assert payload.get("no_contest") is True
|
| 147 |
+
assert "winner" not in payload
|
| 148 |
+
assert payload["text"] # the drama survives β the show always ends
|
| 149 |
+
|
| 150 |
+
def test_missing_winner_is_not_an_error(self):
|
| 151 |
+
# winner is optional; a judge that names none must NOT trigger a re-ask.
|
| 152 |
+
agent, provider = _judge([{"winner": None, "text": "Verdict: the evidence is mute."}])
|
| 153 |
+
payload = _resolve(agent)
|
| 154 |
+
assert provider.calls == 1
|
| 155 |
+
assert payload.get("winner") is None
|
| 156 |
+
assert "no_contest" not in payload
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
class TestScoresNormalisation:
|
| 160 |
+
"""scores is garnish: cleaned in place, never re-asked."""
|
| 161 |
+
|
| 162 |
+
def test_non_cast_keys_dropped_and_values_clamped(self):
|
| 163 |
+
agent, provider = _judge(
|
| 164 |
+
[
|
| 165 |
+
{
|
| 166 |
+
"winner": "clue-gatherer",
|
| 167 |
+
"text": "Verdict: the gatherer cracked it.",
|
| 168 |
+
"scores": {"clue-gatherer": 12, "ghost": 5, "hypothesis-former": -3},
|
| 169 |
+
}
|
| 170 |
+
]
|
| 171 |
+
)
|
| 172 |
+
payload = _resolve(agent)
|
| 173 |
+
assert provider.calls == 1 # scores never cause a re-ask
|
| 174 |
+
# ghost is off-cast β dropped; 12 β clamped to 10; -3 β clamped to 0.
|
| 175 |
+
assert payload["scores"] == {"clue-gatherer": 10.0, "hypothesis-former": 0.0}
|
| 176 |
+
|
| 177 |
+
def test_bad_winner_with_scores_still_only_reasks_for_winner(self):
|
| 178 |
+
agent, provider = _judge(
|
| 179 |
+
[
|
| 180 |
+
{"winner": "bad", "scores": {"clue-gatherer": 99}},
|
| 181 |
+
{
|
| 182 |
+
"winner": "clue-gatherer",
|
| 183 |
+
"text": "Verdict: the gatherer cracked it.",
|
| 184 |
+
"scores": {"clue-gatherer": 7},
|
| 185 |
+
},
|
| 186 |
+
]
|
| 187 |
+
)
|
| 188 |
+
payload = _resolve(agent)
|
| 189 |
+
assert provider.calls == 2
|
| 190 |
+
assert payload["scores"] == {"clue-gatherer": 7.0}
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
class TestHookInert:
|
| 194 |
+
"""The validation hook is a transparent pass-through for everything that is not a
|
| 195 |
+
judge in a live competition β even a downright odd winner slips through untouched."""
|
| 196 |
+
|
| 197 |
+
def test_judged_winner_passes_unchanged(self):
|
| 198 |
+
agent, provider = _judge([{"winner": "devils-advocate", "text": "Verdict: the advocate won."}])
|
| 199 |
+
payload = _resolve(agent)
|
| 200 |
+
assert provider.calls == 1
|
| 201 |
+
assert payload["winner"] == "devils-advocate"
|
| 202 |
+
|
| 203 |
+
def test_non_judge_role_leaves_odd_winner_untouched(self):
|
| 204 |
+
# The hook keys on role == "judge". Take the judge's exact schema (so the
|
| 205 |
+
# winner field and verdict kind stay valid) but flip the role to a worker: the
|
| 206 |
+
# hook must go inert, so an off-cast winner rides through verbatim β no
|
| 207 |
+
# validation, no re-ask, no no_contest.
|
| 208 |
+
reg = default_registry()
|
| 209 |
+
provider = _ScriptedJudge([{"winner": "anything-goes", "text": "Verdict-shaped, but a worker said it."}])
|
| 210 |
+
agent = reg.build_agent("mystery-judge", _ScriptedRouter(provider))
|
| 211 |
+
cfg = reg.scenarios["mystery-roots"]
|
| 212 |
+
agent.competition = cfg.competition
|
| 213 |
+
agent.cast_names = list(cfg.cast)
|
| 214 |
+
agent.manifest = agent.manifest.model_copy(update={"role": "worker"})
|
| 215 |
+
payload = _resolve(agent)
|
| 216 |
+
assert provider.calls == 1 # no re-ask despite the off-cast winner
|
| 217 |
+
assert payload["winner"] == "anything-goes"
|
| 218 |
+
assert "no_contest" not in payload
|
| 219 |
+
|
| 220 |
+
def test_no_competition_attached_leaves_winner_untouched(self):
|
| 221 |
+
# A bare-built judge (no registry injection) has competition=None β hook inert.
|
| 222 |
+
reg = default_registry()
|
| 223 |
+
provider = _ScriptedJudge([{"winner": "off-the-wall", "text": "Verdict: chaos reigns."}])
|
| 224 |
+
agent = reg.build_agent("mystery-judge", _ScriptedRouter(provider))
|
| 225 |
+
# Deliberately do NOT set agent.competition / cast_names.
|
| 226 |
+
assert agent.competition is None
|
| 227 |
+
payload = _resolve(agent)
|
| 228 |
+
assert provider.calls == 1
|
| 229 |
+
assert payload["winner"] == "off-the-wall"
|
| 230 |
+
|
| 231 |
+
def test_none_kind_competition_is_inert(self):
|
| 232 |
+
from src.core.config import CompetitionConfig
|
| 233 |
+
|
| 234 |
+
reg = default_registry()
|
| 235 |
+
provider = _ScriptedJudge([{"winner": "whoever", "text": "Verdict: nobody wins the wood."}])
|
| 236 |
+
agent = reg.build_agent("mystery-judge", _ScriptedRouter(provider))
|
| 237 |
+
agent.competition = CompetitionConfig(kind="none")
|
| 238 |
+
agent.cast_names = ["clue-gatherer"]
|
| 239 |
+
payload = _resolve(agent)
|
| 240 |
+
assert provider.calls == 1
|
| 241 |
+
assert payload["winner"] == "whoever"
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
# ββ offline path: re-ask works there too, and the stub never triggers it βββββββββββ
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
class TestOfflineVerdictPath:
|
| 248 |
+
def test_offline_judge_emits_clean_winnerless_payload_no_reask(self):
|
| 249 |
+
# The stub's _synth_field returns None/β{} for winner/scores, so an offline
|
| 250 |
+
# judge produces a validation-clean payload with no wasted corrective round-trip.
|
| 251 |
+
reg = default_registry()
|
| 252 |
+
agent = reg.build_agent("mystery-judge", ModelRouter(offline=True))
|
| 253 |
+
cfg = reg.scenarios["mystery-roots"]
|
| 254 |
+
agent.competition = cfg.competition
|
| 255 |
+
agent.cast_names = list(cfg.cast)
|
| 256 |
+
payload = _resolve(agent)
|
| 257 |
+
assert payload["winner"] is None
|
| 258 |
+
assert payload["scores"] == {}
|
| 259 |
+
assert "no_contest" not in payload
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
# ββ SpyHost ground-truth scoreboard ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _spy_host() -> SpyHost:
|
| 266 |
+
reg = default_registry()
|
| 267 |
+
host = reg.build_agent("spy-host", ModelRouter(offline=True))
|
| 268 |
+
cfg = reg.scenarios["the-steeped"]
|
| 269 |
+
host.competition = cfg.competition
|
| 270 |
+
host.cast_names = list(cfg.cast)
|
| 271 |
+
return host
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
class TestScanAccusation:
|
| 275 |
+
"""_scan_accusation recovers the accused from verdict text by the distinctive tail
|
| 276 |
+
of each cast name (``spy-cara`` β ``cara``), earliest mention wins, host excluded."""
|
| 277 |
+
|
| 278 |
+
def test_finds_named_player(self):
|
| 279 |
+
host = _spy_host()
|
| 280 |
+
assert host._scan_accusation("Verdict: I point at NIL.") == "spy-nil"
|
| 281 |
+
|
| 282 |
+
def test_matches_case_insensitively(self):
|
| 283 |
+
host = _spy_host()
|
| 284 |
+
assert host._scan_accusation("CARA slipped a tell at dawn.") == "spy-cara"
|
| 285 |
+
|
| 286 |
+
def test_earliest_mention_wins(self):
|
| 287 |
+
host = _spy_host()
|
| 288 |
+
# BEX appears before NIL β bex is the accusation, even though both are named.
|
| 289 |
+
assert host._scan_accusation("BEX hesitated, then NIL steeped too fast.") == "spy-bex"
|
| 290 |
+
|
| 291 |
+
def test_host_token_never_self_accuses(self):
|
| 292 |
+
host = _spy_host()
|
| 293 |
+
# The host's own name token ("host") is excluded; no player named β None.
|
| 294 |
+
assert host._scan_accusation("The host weighs the room and the clues.") is None
|
| 295 |
+
|
| 296 |
+
def test_no_recoverable_name_returns_none(self):
|
| 297 |
+
host = _spy_host()
|
| 298 |
+
assert host._scan_accusation("A long deliberation with no names at all.") is None
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
class TestStampScoreboard:
|
| 302 |
+
"""The load-bearing split: code turns the judge's accusation into the result."""
|
| 303 |
+
|
| 304 |
+
def _verdict_event(self, **payload) -> Event:
|
| 305 |
+
return Event(run_id="r", turn=1, kind="judge.verdict", actor="spy-host", payload=payload)
|
| 306 |
+
|
| 307 |
+
def test_correct_accusation_lets_the_herd_win(self):
|
| 308 |
+
host = _spy_host()
|
| 309 |
+
event = self._verdict_event(text="Verdict: NIL is the spy.", winner="spy-nil")
|
| 310 |
+
host._stamp_scoreboard(event)
|
| 311 |
+
assert event.payload["accused"] == "spy-nil"
|
| 312 |
+
assert event.payload["correct"] is True
|
| 313 |
+
assert event.payload["winner"] == "herd" # spy caught β herd wins
|
| 314 |
+
|
| 315 |
+
def test_wrong_accusation_lets_the_spy_win(self):
|
| 316 |
+
host = _spy_host()
|
| 317 |
+
event = self._verdict_event(text="Verdict: CARA is the spy.", winner="spy-cara")
|
| 318 |
+
host._stamp_scoreboard(event)
|
| 319 |
+
assert event.payload["accused"] == "spy-cara"
|
| 320 |
+
assert event.payload["correct"] is False
|
| 321 |
+
assert event.payload["winner"] == "spy" # innocent accused β spy wins
|
| 322 |
+
|
| 323 |
+
def test_offline_accusation_recovered_from_text(self):
|
| 324 |
+
# No winner field (the offline shape) β the accusation is scanned from text.
|
| 325 |
+
host = _spy_host()
|
| 326 |
+
event = self._verdict_event(text="Verdict: I point at NIL. The herd's clues brewed.")
|
| 327 |
+
host._stamp_scoreboard(event)
|
| 328 |
+
assert event.payload["accused"] == "spy-nil"
|
| 329 |
+
assert event.payload["winner"] == "herd"
|
| 330 |
+
|
| 331 |
+
def test_no_recoverable_accusation_is_no_contest(self):
|
| 332 |
+
host = _spy_host()
|
| 333 |
+
event = self._verdict_event(text="A long deliberation with no names at all.")
|
| 334 |
+
host._stamp_scoreboard(event)
|
| 335 |
+
assert event.payload.get("no_contest") is True
|
| 336 |
+
assert "winner" not in event.payload
|
| 337 |
+
|
| 338 |
+
def test_no_spy_team_is_inert(self):
|
| 339 |
+
# A versus competition without a 'spy' team has no ground truth to stamp.
|
| 340 |
+
from src.core.config import CompetitionConfig
|
| 341 |
+
|
| 342 |
+
host = _spy_host()
|
| 343 |
+
host.competition = CompetitionConfig(kind="versus", teams={"a": ["spy-cara"], "b": ["spy-bex"]})
|
| 344 |
+
event = self._verdict_event(text="Verdict: NIL.", winner="spy-nil")
|
| 345 |
+
host._stamp_scoreboard(event)
|
| 346 |
+
# Untouched: no accused/correct/no_contest stamped, winner left as the raw pick.
|
| 347 |
+
assert "accused" not in event.payload
|
| 348 |
+
assert "correct" not in event.payload
|
| 349 |
+
assert event.payload["winner"] == "spy-nil"
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
class TestSpyHostActEnrichment:
|
| 353 |
+
"""The full handler turn: super().act() produces the verdict, then the scoreboard
|
| 354 |
+
is stamped β driven entirely offline through the deterministic stub."""
|
| 355 |
+
|
| 356 |
+
def test_act_stamps_a_full_scoreboard_offline(self):
|
| 357 |
+
host = _spy_host()
|
| 358 |
+
# The stub's spy-host lines all name NIL β a deterministic herd win.
|
| 359 |
+
recent = tuple(
|
| 360 |
+
Event(run_id="r", turn=1, kind="agent.spoke", actor=p, payload={"text": "a clue"})
|
| 361 |
+
for p in ("spy-cara", "spy-bex", "spy-nil", "spy-ovo")
|
| 362 |
+
)
|
| 363 |
+
projection = StageProjection()
|
| 364 |
+
event = host.act("r", 2, projection, recent)
|
| 365 |
+
assert event.kind == "judge.verdict"
|
| 366 |
+
assert event.payload["accused"] == "spy-nil"
|
| 367 |
+
assert event.payload["correct"] is True
|
| 368 |
+
assert event.payload["winner"] == "herd"
|
| 369 |
+
# The dramatic reveal still rides alongside the scoreboard.
|
| 370 |
+
assert isinstance(event.payload.get("reveal"), list) and event.payload["reveal"]
|