feat(observability): instrument registry, tools, session, modal (Units 7,8,10,11)

- registry: config.loaded, manifest.loaded, model-endpoint resolution (no api key),
cast assembly, governor configuration logs.
- tools: tool.call span + capability-check (granted/denied) logs + tool.calls metric.
- session: session.created/reset/step/inject logs + session.step span so a UI step
produces a full trace down through the conductor.
- modal: configure() + modal.run_episode span + episode start/done logs with token stats.

Co-Authored-By: Codex <codex@openai.com>

modal_app.py

@@ -8,6 +8,7 @@ used only when you deploy it:
     modal run    modal_app.py        # one-off episode
     modal deploy modal_app.py        # schedule it (hourly by default)
 """
+
 from __future__ import annotations
 
 import modal
@@ -38,12 +39,14 @@ def run_episode(scenario_name: str = DEFAULT_SCENARIO, n_ticks: int = TICKS_PER_
     import os
     from pathlib import Path
 
+    from src import observability as obs
     from src.core.conductor import Conductor
     from src.core.ledger_factory import database_url
     from src.core.registry import default_registry
     from src.core.sqlite_ledger import SQLiteLedger
     from src.tools.builtins import default_tool_registry
 
+    obs.configure()
     db_path = f"/data/{scenario_name}.db"
     reg = default_registry()
     # Durable event store when DATABASE_URL is set (ADR-0014); otherwise the
@@ -62,12 +65,23 @@ def run_episode(scenario_name: str = DEFAULT_SCENARIO, n_ticks: int = TICKS_PER_
         snapshot_path=f"/data/{scenario_name}.snapshot.db",
     )
 
-    if not conductor.restore():
-        conductor.reset(conductor.scenario.default_seed)
-    conductor.step(n_ticks=n_ticks)
-    ledger.close()
-    volume.commit()  # persist the ledger for the next scheduled run
-    return {"scenario": scenario_name, "turn": conductor.turn, "stats": conductor.governor.stats}
+    with obs.span("modal.run_episode", **{"scenario": scenario_name, "modal.n_ticks": n_ticks}):
+        restored = conductor.restore()
+        if not restored:
+            conductor.reset(conductor.scenario.default_seed)
+        obs.log(
+            "modal.episode.start",
+            scenario=scenario_name,
+            n_ticks=n_ticks,
+            restored=restored,
+            durable=bool(database_url()),
+        )
+        conductor.step(n_ticks=n_ticks)
+        ledger.close()
+        volume.commit()  # persist the ledger for the next scheduled run
+        stats = conductor.governor.stats
+        obs.log("modal.episode.done", scenario=scenario_name, turn=conductor.turn, stats=stats)
+        return {"scenario": scenario_name, "turn": conductor.turn, "stats": stats}
 
 
 @app.local_entrypoint()

src/core/registry.py

@@ -21,6 +21,7 @@ from pathlib import Path
 
 import yaml
 
+from src import observability as obs
 from src.agents.base import Agent, ManifestAgent
 from src.core.config import (
     GovernorConfig,
@@ -103,6 +104,12 @@ def _resolve_model_endpoints(raw_models: dict, env: dict[str, str] | None = None
             cfg.setdefault("model", binding["model"])
         cfg.setdefault("base_url", binding["base_url"])
         cfg.setdefault("api_key", binding["api_key"])
+        obs.log(
+            "registry.model_endpoint",
+            profile=str(profile),
+            model=cfg.get("model", ""),
+            base_url=cfg.get("base_url", ""),
+        )  # never logs api_key
     return raw_models
 
 
@@ -146,6 +153,14 @@ class Registry:
             for path in sorted(agents_dir.glob("*.yaml")):
                 manifest = validate_agent(yaml.safe_load(path.read_text()) or {})
                 agents[manifest.name] = manifest
+                obs.log(
+                    "manifest.loaded",
+                    name=manifest.name,
+                    profile=manifest.model_profile,
+                    endpoint=manifest.model_endpoint or "",
+                    subscribes=len(manifest.subscribes_to),
+                    may_emit=list(manifest.may_emit),
+                )
 
         scenarios: dict[str, ScenarioConfig] = {}
         scenarios_dir = root / "scenarios"
@@ -161,6 +176,7 @@ class Registry:
             raw_models = _resolve_model_endpoints(raw_models)
             models = ModelsConfig.model_validate(raw_models)
 
+        obs.log("config.loaded", config_dir=str(root), agents=len(agents), scenarios=len(scenarios))
         return cls(agents=agents, scenarios=scenarios, models=models)
 
     @classmethod
@@ -214,6 +230,13 @@ class Registry:
 
         memory_index = memory_index_from_env()
         agents = tuple(self.build_agent(agent_name, router, tools, memory_index) for agent_name in cfg.cast)
+        obs.log(
+            "registry.cast_assembled",
+            scenario=cfg.name,
+            cast=list(cfg.cast),
+            count=len(cfg.cast),
+            offline=getattr(router, "offline", None),
+        )
         return Scenario(
             name=cfg.name,
             default_seed=cfg.default_seed,
@@ -227,6 +250,15 @@ class Registry:
         """Build a Governor from a scenario's budget config (or defaults)."""
         cfg = self.scenarios.get(name)
         budget = (cfg.governor if cfg else None) or GovernorConfig()
+        obs.log(
+            "governor.configured",
+            scenario=name,
+            max_turns=budget.max_turns,
+            max_calls_per_turn=budget.max_calls_per_turn,
+            max_total_calls=budget.max_total_calls,
+            max_total_tokens=budget.max_total_tokens,
+            hourly_budget_usd=budget.hourly_budget_usd,
+        )
         return Governor(
             max_turns=budget.max_turns,
             max_calls_per_turn=budget.max_calls_per_turn,

src/tools/registry.py

@@ -10,12 +10,14 @@ returns a JSON-serialisable dict that the calling agent folds into its event.
 In-process callables and MCP-server-backed tools both satisfy this interface, so
 swapping a local stub for a real MCP server is invisible to agents.
 """
+
 from __future__ import annotations
 
 from collections.abc import Callable
 from dataclasses import dataclass
 from typing import Protocol
 
+from src import observability as obs
 from src.core.manifest import AgentManifest
 
 
@@ -93,13 +95,21 @@ class ToolRegistry:
         precedence; otherwise the call is dispatched to the resolver if one backs
         the tool.  An unknown granted tool raises :class:`KeyError` as before.
         """
-        if tool not in manifest.tools:
-            raise CapabilityViolation(
-                f"{agent_name!r} is not authorised to call tool {tool!r} "
-                f"(granted: {manifest.tools})"
-            )
-        if tool in self._tools:
-            return self._tools[tool].run(**params)
-        if self._resolver is not None and self._resolver.has(tool):
-            return self._resolver.call(tool, params)
-        raise KeyError(f"unknown tool {tool!r} (registered: {sorted(self._tools)})")
+        with obs.span("tool.call", **{"tool": tool, "mal.agent": agent_name, "tool.params": sorted(params)}):
+            if tool not in manifest.tools:
+                obs.log("tool.denied", level="warning", agent=agent_name, tool=tool, granted=list(manifest.tools))
+                raise CapabilityViolation(
+                    f"{agent_name!r} is not authorised to call tool {tool!r} (granted: {manifest.tools})"
+                )
+            obs.incr("tool.calls", 1, tool=tool)
+            if tool in self._tools:
+                result = self._tools[tool].run(**params)
+                obs.log("tool.call", agent=agent_name, tool=tool, transport="in-process")
+                obs.log("tool.result", level="debug", tool=tool, result=result)
+                return result
+            if self._resolver is not None and self._resolver.has(tool):
+                result = self._resolver.call(tool, params)
+                obs.log("tool.call", agent=agent_name, tool=tool, transport="resolver")
+                obs.log("tool.result", level="debug", tool=tool, result=result)
+                return result
+            raise KeyError(f"unknown tool {tool!r} (registered: {sorted(self._tools)})")

src/ui/fishbowl/session.py

@@ -12,6 +12,7 @@ cast, so a session is reproducible on stage.
 
 from __future__ import annotations
 
+from src import observability as obs
 from src.core.conductor import Conductor
 from src.core.ledger_factory import make_ledger
 from src.core.manifest import AgentManifest
@@ -49,21 +50,33 @@ class FishbowlSession:
             governor=self._registry.governor_for(scenario_name),
             ledger=make_ledger(),
         )
+        obs.log(
+            "session.created",
+            scenario=scenario_name,
+            ledger=type(self.conductor.ledger).__name__,
+            cast=len(scenario.agents),
+        )
 
     # ── lifecycle ───────────────────────────────────────────────────────────────
 
     def reset(self, seed: str = "") -> None:
+        obs.log("session.reset", scenario=self._scenario_name, seed=seed or self.scenario.default_seed)
         self.conductor.reset(seed or self.scenario.default_seed)
 
     def step(self, n_ticks: int = 1) -> None:
-        self.conductor.step(n_ticks)
+        with obs.span("session.step", **{"session.n_ticks": n_ticks}):
+            obs.log("session.step", n_ticks=n_ticks)
+            self.conductor.step(n_ticks)
 
     def step_one(self) -> bool:
         """Advance a single agent (streaming): one event per call so the Show reveals
         each mind the moment it responds, not after the whole turn finishes."""
-        return self.conductor.step_one()
+        with obs.span("session.step", **{"session.streaming": True}):
+            return self.conductor.step_one()
 
     def inject(self, text: str, label: str | None = None) -> None:
+        obs.log("session.inject", label=label or "", chars=len(text))
+        obs.log("session.inject.text", level="debug", label=label or "", text=text)
         self.conductor.inject_user_event(text, label=label)
         self.conductor.step()