feat: refine structured output handling with guided decoding and mode adjustments

docs/adr/0016-instructor-structured-output.md

@@ -66,6 +66,28 @@ recorded from the provider in every branch, so the conductor's
 `pyproject.toml`. Lazy imports keep `import src.*` and `import app` working with
 it not installed.
 
+## Refinement: guided decoding, not tool calling (2026-06)
+
+The first cut left Instructor on its default `Mode.TOOLS`, which encodes the
+schema as an OpenAI **function/tool call**. That only validates on a served model
+whose vLLM deployment has tool calling enabled with a *matching* parser. The
+`fast` tier (`minicpm-4-1-8b`, ADR-0022 catalogue) has neither: MiniCPM4.1 emits a
+custom `<|tool_call_start|> … <|tool_call_end|>` format for which vLLM 0.21.0 ships
+no parser, so every structured call returned **`400 Bad Request`** (rejected at
+request validation, ~40 ms, no generation) and degraded to the prose fallback —
+turning the `fast` tier's fast validated-JSON path into a ~7 s prose round-trip
+every turn, and feeding the `clean_clue` over-filter that dropped first-person
+clues (the `spy-bex` "no usable line" failure).
+
+`LiteLLMProvider.structured_mode` now defaults to **`json_schema`** — vLLM
+**guided decoding** via `response_format`, which constrains output to the schema
+*without* a tool-call parser, so it is correct for every served model regardless of
+tool support (Gemma/Nemotron keep validating; MiniCPM now validates instead of
+400ing). It is a per-provider field (an `instructor.Mode` member name): `json`
+(plain `json_object` + schema-in-prompt) is the fallback if a backend rejects
+`json_schema`, and `tools` restores the old behaviour for a model that prefers it.
+No redeploy is needed — the change is entirely client-side on the request shape.
+
 ## Consequences
 
 - On the live path, agent output is schema-valid and kind-constrained by

modal/catalogue.py

@@ -197,6 +197,11 @@ OPENBMB_MODELS: tuple[ModelConfig, ...] = (
         max_model_len=32768,
         trust_remote_code=True,
         max_concurrent_inputs=48,
+        # No tool_call_parser on purpose: MiniCPM4.1 emits a custom
+        # <|tool_call_start|> format vLLM 0.21.0 has no parser for, so tool-call
+        # structured output 400s here. The engine's structured path uses vLLM
+        # guided decoding (response_format json_schema) instead, which is
+        # parser-independent — see ADR-0016. Don't bolt on a mismatched parser.
     ),
     ModelConfig(
         name="openbmb/MiniCPM-o-4_5",

tests/test_instructor.py

@@ -105,8 +105,12 @@ class _FakeInstructorClient:
         return result, _FakeRawCompletion(hidden_cost=self._hidden_cost)
 
 
-def _install_fakes(monkeypatch, *, client) -> None:
-    """Inject fake ``instructor`` (from_litellm -> client) and ``litellm`` modules."""
+def _install_fakes(monkeypatch, *, client, from_litellm_kw: dict | None = None) -> None:
+    """Inject fake ``instructor`` (from_litellm -> client) and ``litellm`` modules.
+
+    *from_litellm_kw*, when given, records the kwargs ``complete_structured`` passes to
+    ``instructor.from_litellm`` (e.g. the chosen ``mode``) for assertion.
+    """
     fake_litellm = types.ModuleType("litellm")
     fake_litellm.completion = lambda **kw: None
 
@@ -115,8 +119,16 @@ def _install_fakes(monkeypatch, *, client) -> None:
 
     fake_litellm.completion_cost = _completion_cost
 
+    def _from_litellm(completion, **kw):
+        if from_litellm_kw is not None:
+            from_litellm_kw.update(kw)
+        return client
+
     fake_instructor = types.ModuleType("instructor")
-    fake_instructor.from_litellm = lambda completion, **kw: client
+    fake_instructor.from_litellm = _from_litellm
+    # Mode is an enum on the real package; a name->value stand-in is enough for the
+    # provider's ``getattr(instructor.Mode, structured_mode.upper())`` resolution.
+    fake_instructor.Mode = types.SimpleNamespace(JSON_SCHEMA="json_schema", JSON="json", TOOLS="tools")
 
     monkeypatch.setitem(sys.modules, "litellm", fake_litellm)
     monkeypatch.setitem(sys.modules, "instructor", fake_instructor)
@@ -156,6 +168,22 @@ class TestCompleteStructured:
         roles = [m["role"] for m in record["messages"]]
         assert roles == ["system", "user"]
 
+    def test_defaults_to_guided_json_schema_mode(self, monkeypatch):
+        # Guided decoding, not tool calling: a model with no tool-call parser (e.g. MiniCPM)
+        # still validates instead of 400ing. The mode rides on from_litellm, not the call.
+        kw: dict = {}
+        _install_fakes(monkeypatch, client=_FakeInstructorClient(), from_litellm_kw=kw)
+        provider = LiteLLMProvider(model="openai/m", api_base="https://x/v1")
+        provider.complete_structured("echo", "x", build_output_model(["agent.spoke"]))
+        assert kw["mode"] == "json_schema"
+
+    def test_structured_mode_override_is_honored(self, monkeypatch):
+        kw: dict = {}
+        _install_fakes(monkeypatch, client=_FakeInstructorClient(), from_litellm_kw=kw)
+        provider = LiteLLMProvider(model="openai/m", api_base="https://x/v1", structured_mode="tools")
+        provider.complete_structured("echo", "x", build_output_model(["agent.spoke"]))
+        assert kw["mode"] == "tools"
+
     def test_error_zeroes_usage_and_reraises(self, monkeypatch):
         _install_fakes(monkeypatch, client=_FakeInstructorClient(raise_exc=RuntimeError("boom")))
         provider = LiteLLMProvider(model="openai/m", api_base="https://x/v1")