import type { AssistantMessage } from "@mariozechner/pi-ai"; import { describe, expect, it } from "vitest"; import { formatBillingErrorMessage } from "../../pi-embedded-helpers.js"; import { makeAssistantMessageFixture } from "../../test-helpers/assistant-message-fixtures.js"; import { buildPayloads, expectSinglePayloadText, expectSingleToolErrorPayload, } from "./payloads.test-helpers.js"; describe("buildEmbeddedRunPayloads", () => { const OVERLOADED_FALLBACK_TEXT = "The AI service is temporarily overloaded. Please try again in a moment."; const errorJson = '{"type":"error","error":{"details":null,"type":"overloaded_error","message":"Overloaded"},"request_id":"req_011CX7DwS7tSvggaNHmefwWg"}'; const errorJsonPretty = `{ "type": "error", "error": { "details": null, "type": "overloaded_error", "message": "Overloaded" }, "request_id": "req_011CX7DwS7tSvggaNHmefwWg" }`; const makeAssistant = (overrides: Partial): AssistantMessage => makeAssistantMessageFixture({ errorMessage: errorJson, content: [{ type: "text", text: errorJson }], ...overrides, }); const makeStoppedAssistant = () => makeAssistant({ stopReason: "stop", errorMessage: undefined, content: [], }); const expectOverloadedFallback = (payloads: ReturnType) => { expect(payloads).toHaveLength(1); expect(payloads[0]?.text).toBe(OVERLOADED_FALLBACK_TEXT); }; function expectNoSyntheticCompletionForSession(sessionKey: string) { const payloads = buildPayloads({ sessionKey, toolMetas: [{ toolName: "write", meta: "/tmp/out.md" }], lastAssistant: makeAssistant({ stopReason: "stop", errorMessage: undefined, content: [], }), }); expect(payloads).toHaveLength(0); } it("suppresses raw API error JSON when the assistant errored", () => { const payloads = buildPayloads({ assistantTexts: [errorJson], lastAssistant: makeAssistant({}), }); expectOverloadedFallback(payloads); expect(payloads[0]?.isError).toBe(true); expect(payloads.some((payload) => payload.text === errorJson)).toBe(false); }); it("suppresses pretty-printed error JSON that differs from the errorMessage", () => { const payloads = buildPayloads({ assistantTexts: [errorJsonPretty], lastAssistant: makeAssistant({ errorMessage: errorJson }), inlineToolResultsAllowed: true, verboseLevel: "on", }); expectOverloadedFallback(payloads); expect(payloads.some((payload) => payload.text === errorJsonPretty)).toBe(false); }); it("suppresses raw error JSON from fallback assistant text", () => { const payloads = buildPayloads({ lastAssistant: makeAssistant({ content: [{ type: "text", text: errorJsonPretty }] }), }); expectOverloadedFallback(payloads); expect(payloads.some((payload) => payload.text?.includes("request_id"))).toBe(false); }); it("includes provider and model context for billing errors", () => { const payloads = buildPayloads({ lastAssistant: makeAssistant({ model: "claude-3-5-sonnet", errorMessage: "insufficient credits", content: [{ type: "text", text: "insufficient credits" }], }), provider: "Anthropic", model: "claude-3-5-sonnet", }); expect(payloads).toHaveLength(1); expect(payloads[0]?.text).toBe(formatBillingErrorMessage("Anthropic", "claude-3-5-sonnet")); expect(payloads[0]?.isError).toBe(true); }); it("does not emit a synthetic billing error for successful turns with stale errorMessage", () => { const payloads = buildPayloads({ lastAssistant: makeAssistant({ stopReason: "stop", errorMessage: "insufficient credits for embedding model", content: [{ type: "text", text: "Handle payment required errors in your API." }], }), }); expectSinglePayloadText(payloads, "Handle payment required errors in your API."); }); it("suppresses raw error JSON even when errorMessage is missing", () => { const payloads = buildPayloads({ assistantTexts: [errorJsonPretty], lastAssistant: makeAssistant({ errorMessage: undefined }), }); expect(payloads).toHaveLength(1); expect(payloads[0]?.isError).toBe(true); expect(payloads.some((payload) => payload.text?.includes("request_id"))).toBe(false); }); it("does not suppress error-shaped JSON when the assistant did not error", () => { const payloads = buildPayloads({ assistantTexts: [errorJsonPretty], lastAssistant: makeStoppedAssistant(), }); expectSinglePayloadText(payloads, errorJsonPretty.trim()); }); it("adds a fallback error when a tool fails and no assistant output exists", () => { const payloads = buildPayloads({ lastToolError: { toolName: "browser", error: "tab not found" }, }); expectSingleToolErrorPayload(payloads, { title: "Browser", absentDetail: "tab not found", }); }); it("does not add tool error fallback when assistant output exists", () => { const payloads = buildPayloads({ assistantTexts: ["All good"], lastAssistant: makeStoppedAssistant(), lastToolError: { toolName: "browser", error: "tab not found" }, }); expectSinglePayloadText(payloads, "All good"); }); it("does not add synthetic completion text when tools run without final assistant text", () => { const payloads = buildPayloads({ sessionKey: "agent:main:discord:direct:u123", toolMetas: [{ toolName: "write", meta: "/tmp/out.md" }], lastAssistant: makeStoppedAssistant(), }); expect(payloads).toHaveLength(0); }); it("does not add synthetic completion text for channel sessions", () => { expectNoSyntheticCompletionForSession("agent:main:discord:channel:c123"); }); it("does not add synthetic completion text for group sessions", () => { expectNoSyntheticCompletionForSession("agent:main:telegram:group:g123"); }); it("does not add synthetic completion text when messaging tool already delivered output", () => { const payloads = buildPayloads({ sessionKey: "agent:main:discord:direct:u123", toolMetas: [{ toolName: "message_send", meta: "sent to #ops" }], didSendViaMessagingTool: true, lastAssistant: makeAssistant({ stopReason: "stop", errorMessage: undefined, content: [], }), }); expect(payloads).toHaveLength(0); }); it("suppresses non-mutating tool warnings when messaging tool already delivered the reply", () => { const payloads = buildPayloads({ toolMetas: [{ toolName: "message_send", meta: "sent to #ops" }], didSendViaMessagingTool: true, lastToolError: { toolName: "browser", error: "connection timeout" }, }); expect(payloads).toHaveLength(0); }); it("does not add synthetic completion text when the run still has a tool error", () => { const payloads = buildPayloads({ toolMetas: [{ toolName: "browser", meta: "open https://example.com" }], lastToolError: { toolName: "browser", error: "url required" }, }); expect(payloads).toHaveLength(0); }); it("does not add synthetic completion text when no tools ran", () => { const payloads = buildPayloads({ lastAssistant: makeStoppedAssistant(), }); expect(payloads).toHaveLength(0); }); it("adds tool error fallback when the assistant only invoked tools and verbose mode is on", () => { const payloads = buildPayloads({ lastAssistant: makeAssistant({ stopReason: "toolUse", errorMessage: undefined, content: [ { type: "toolCall", id: "toolu_01", name: "exec", arguments: { command: "echo hi" }, }, ], }), lastToolError: { toolName: "exec", error: "Command exited with code 1" }, verboseLevel: "on", }); expectSingleToolErrorPayload(payloads, { title: "Exec", detail: "code 1", }); }); it("does not add tool error fallback when assistant text exists after tool calls", () => { const payloads = buildPayloads({ assistantTexts: ["Checked the page and recovered with final answer."], lastAssistant: makeAssistant({ stopReason: "toolUse", errorMessage: undefined, content: [ { type: "toolCall", id: "toolu_01", name: "browser", arguments: { action: "search", query: "openclaw docs" }, }, ], }), lastToolError: { toolName: "browser", error: "connection timeout" }, }); expect(payloads).toHaveLength(1); expect(payloads[0]?.isError).toBeUndefined(); expect(payloads[0]?.text).toContain("recovered"); }); it("suppresses recoverable tool errors containing 'required' for non-mutating tools", () => { const payloads = buildPayloads({ lastToolError: { toolName: "browser", error: "url required" }, }); // Recoverable errors should not be sent to the user expect(payloads).toHaveLength(0); }); it("suppresses recoverable tool errors containing 'missing' for non-mutating tools", () => { const payloads = buildPayloads({ lastToolError: { toolName: "browser", error: "url missing" }, }); expect(payloads).toHaveLength(0); }); it("suppresses recoverable tool errors containing 'invalid' for non-mutating tools", () => { const payloads = buildPayloads({ lastToolError: { toolName: "browser", error: "invalid parameter: url" }, }); expect(payloads).toHaveLength(0); }); it("suppresses non-mutating non-recoverable tool errors when messages.suppressToolErrors is enabled", () => { const payloads = buildPayloads({ lastToolError: { toolName: "browser", error: "connection timeout" }, config: { messages: { suppressToolErrors: true } }, }); expect(payloads).toHaveLength(0); }); it("suppresses mutating tool errors when suppressToolErrorWarnings is enabled", () => { const payloads = buildPayloads({ lastToolError: { toolName: "exec", error: "command not found" }, suppressToolErrorWarnings: true, }); expect(payloads).toHaveLength(0); }); it.each([ { name: "still shows mutating tool errors when messages.suppressToolErrors is enabled", payload: { lastToolError: { toolName: "write", error: "connection timeout" }, config: { messages: { suppressToolErrors: true } }, }, title: "Write", absentDetail: "connection timeout", }, { name: "shows recoverable tool errors for mutating tools", payload: { lastToolError: { toolName: "message", meta: "reply", error: "text required" }, }, title: "Message", absentDetail: "required", }, { name: "shows non-recoverable tool failure summaries to the user", payload: { lastToolError: { toolName: "browser", error: "connection timeout" }, }, title: "Browser", absentDetail: "connection timeout", }, ])("$name", ({ payload, title, absentDetail }) => { const payloads = buildPayloads(payload); expectSingleToolErrorPayload(payloads, { title, absentDetail }); }); it("shows mutating tool errors even when assistant output exists", () => { const payloads = buildPayloads({ assistantTexts: ["Done."], lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, lastToolError: { toolName: "write", error: "file missing" }, }); expect(payloads).toHaveLength(2); expect(payloads[0]?.text).toBe("Done."); expect(payloads[1]?.isError).toBe(true); expect(payloads[1]?.text).toContain("Write"); expect(payloads[1]?.text).not.toContain("missing"); }); it("does not treat session_status read failures as mutating when explicitly flagged", () => { const payloads = buildPayloads({ assistantTexts: ["Status loaded."], lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, lastToolError: { toolName: "session_status", error: "model required", mutatingAction: false, }, }); expect(payloads).toHaveLength(1); expect(payloads[0]?.text).toBe("Status loaded."); }); it("dedupes identical tool warning text already present in assistant output", () => { const seed = buildPayloads({ lastToolError: { toolName: "write", error: "file missing", mutatingAction: true, }, }); const warningText = seed[0]?.text; expect(warningText).toBeTruthy(); const payloads = buildPayloads({ assistantTexts: [warningText ?? ""], lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, lastToolError: { toolName: "write", error: "file missing", mutatingAction: true, }, }); expect(payloads).toHaveLength(1); expect(payloads[0]?.text).toBe(warningText); }); it("includes non-recoverable tool error details when verbose mode is on", () => { const payloads = buildPayloads({ lastToolError: { toolName: "browser", error: "connection timeout" }, verboseLevel: "on", }); expectSingleToolErrorPayload(payloads, { title: "Browser", detail: "connection timeout", }); }); });