import { sanitizeForLog } from "../../../packages/terminal-core/src/ansi.js"; import { classifyOAuthRefreshFailure, classifyOAuthRefreshFailureError, } from "../../agents/auth-profiles/oauth-refresh-failure.js"; import { isCompactionFailureError, isLikelyContextOverflowError, } from "../../agents/embedded-agent-helpers.js"; import { findCliTimeoutError, isFailoverError } from "../../agents/failover-error.js"; import { GENERIC_EXTERNAL_RUN_FAILURE_TEXT, HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT, renderControlUiAgentFailureCopy, renderFailoverCodeUserCopy, } from "../../agents/failover/user-copy.js"; import { isAgentHarnessPreflightError } from "../../agents/harness/errors.js"; import { LiveSessionModelSwitchError } from "../../agents/live-model-switch-error.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { CommandLaneClearedError, GatewayDrainingError } from "../../process/command-queue.js"; import { defaultRuntime } from "../../runtime.js"; import type { ReplyPayload } from "../types.js"; import { createAgentLifecycleTerminalBackstop } from "./agent-lifecycle-terminal.js"; import { buildContextOverflowRecoveryText } from "./agent-runner-context-recovery.js"; import type { AgentTurnInternalResult, AgentTurnParams } from "./agent-runner-execution.types.js"; import { buildAuthProfileFailoverFailureText, buildExternalRunFailureReply, isNonDirectConversationContext, isVerboseFailureDetailEnabled, markAgentRunFailureReplyPayload, resolveExternalRunFailureTextForConversation, resolveReplyFailureSummary, resolveReplyFailoverFacts, } from "./agent-runner-failure-reply.js"; import type { AgentFallbackCycleState } from "./agent-runner-fallback-cycle.js"; import type { AgentTurnTimingTracker } from "./agent-runner-turn-timing.js"; import { buildRestartLifecycleReplyText, resolveReplyOperationAbortReason, resolveReplyOperationTerminationFields, resolveRestartLifecycleError, } from "./reply-operation-abort.js"; const MAX_LIVE_SWITCH_RETRIES = 2; type ErrorAction = | { kind: "retry"; liveModelSwitchError?: LiveSessionModelSwitchError } | Extract; export async function handleAgentExecutionError(params: { turn: AgentTurnParams; error: unknown; runtimeConfig: AgentTurnParams["followupRun"]["run"]["config"]; runId: string; state: AgentFallbackCycleState; liveModelSwitchRetries: number; shouldSurfaceToControlUi: boolean; timing: AgentTurnTimingTracker; modelPatch: { fail: (error: unknown) => Promise }; }): Promise { const turn = params.turn; const err = params.error; // A failed candidate leaves its backstop pending; settlement takes it before later work. // This keeps session-override failures from being mislabeled as model failures. const postCompactionModelFailure = params.state.postCompactionModelAttempted && params.state.pendingLifecycleTerminal ? true : undefined; const takePendingLifecycleTerminal = () => { const terminal = params.state.pendingLifecycleTerminal?.backstop ?? createAgentLifecycleTerminalBackstop({ runId: params.runId, sessionKey: turn.sessionKey, startedAt: params.state.turnStartedAtMs, getLifecycleGeneration: () => params.state.lifecycleGeneration, resolveTerminationFields: (error) => resolveReplyOperationTerminationFields( error, turn.replyOperation?.abortSignal ?? turn.opts?.abortSignal, turn.replyOperation, ), }); params.state.pendingLifecycleTerminal = undefined; return terminal; }; const settleFailure = async ( payload: ReplyPayload, ): Promise> => { takePendingLifecycleTerminal().emit("error", err); turn.replyOperation?.fail("run_failed", err); await params.modelPatch.fail(err); return { kind: "final", payload: markAgentRunFailureReplyPayload(payload), postCompactionModelFailure, }; }; const resolveReplyOperationAbortAction = (abortError: unknown): ErrorAction | undefined => { const reason = resolveReplyOperationAbortReason(turn.replyOperation, abortError); if (!reason) { return undefined; } // Preserve signal-owned timeout attribution; only normalized restart/supersession need metadata. const terminalMetadata = reason === "user" ? undefined : { aborted: true, stopReason: reason }; takePendingLifecycleTerminal().emit( reason === "restart" ? "end" : "error", abortError, terminalMetadata, ); return { kind: "aborted", reason }; }; const replyOperationAbortAction = resolveReplyOperationAbortAction(err); if (replyOperationAbortAction) { return replyOperationAbortAction; } if (err instanceof LiveSessionModelSwitchError) { if (params.liveModelSwitchRetries <= MAX_LIVE_SWITCH_RETRIES) { params.state.pendingLifecycleTerminal = undefined; return { kind: "retry", liveModelSwitchError: err }; } const visibleReplyDelivered = await turn.resolveVisibleReplyDelivery?.(); defaultRuntime.error( `Live model switch failed after ${MAX_LIVE_SWITCH_RETRIES} retries ` + `(${sanitizeForLog(err.provider)}/${sanitizeForLog(err.model)}). The requested model may be unavailable.`, ); takePendingLifecycleTerminal().emit("error", err); const switchErrorText = params.shouldSurfaceToControlUi ? renderControlUiAgentFailureCopy( "model switch could not be completed. The requested model may be temporarily unavailable.", ) : isVerboseFailureDetailEnabled(turn.resolvedVerboseLevel) ? "⚠️ Agent failed before reply: model switch could not be completed. " + "The requested model may be temporarily unavailable. Please try again shortly." : "⚠️ Model switch could not be completed. The requested model may be temporarily unavailable. Please try again shortly."; turn.replyOperation?.fail("run_failed", err); await params.modelPatch.fail(err); return { kind: "final", payload: markAgentRunFailureReplyPayload({ text: resolveExternalRunFailureTextForConversation({ text: switchErrorText, visibleReplyDelivered, sessionCtx: turn.sessionCtx, isGenericRunnerFailure: !params.shouldSurfaceToControlUi, cfg: turn.followupRun.run.config, }), }), }; } const message = formatErrorMessage(err); params.timing.logIfSlow({ runId: params.runId, sessionId: turn.followupRun.run.sessionId, sessionKey: turn.sessionKey, outcome: "error", error: message, }); // The exhausted preflight is deliberate, even if its diagnostic cause looks // like HTTP/overload. Settle delivery and normal diagnostic policy without replay. if (isAgentHarnessPreflightError(err)) { const externalReply = buildExternalRunFailureReply( { message, error: err }, { includeDetails: isVerboseFailureDetailEnabled(turn.resolvedVerboseLevel), isHeartbeat: turn.isHeartbeat, }, ); const text = resolveExternalRunFailureTextForConversation({ text: params.shouldSurfaceToControlUi ? renderControlUiAgentFailureCopy(message) : externalReply.text, visibleReplyDelivered: await turn.resolveVisibleReplyDelivery?.(), sessionCtx: turn.sessionCtx, isGenericRunnerFailure: externalReply.isGenericRunnerFailure, cfg: turn.followupRun.run.config, }); return await settleFailure({ text }); } const failoverFacts = resolveReplyFailoverFacts(err, message); const failureSummary = resolveReplyFailureSummary({ error: err, message, reason: failoverFacts.reason, attempts: isFailoverError(err) ? err.attempts : undefined, }); const failoverReason = failoverFacts.reason; const isBilling = failureSummary?.kind === "billing"; const isContextOverflow = !isBilling && (failoverReason === "context_overflow" || isLikelyContextOverflowError(message)); const isCompactionFailure = !isBilling && isCompactionFailureError(message); const oauthRefreshFailure = classifyOAuthRefreshFailureError(err) ?? classifyOAuthRefreshFailure(message); const hasAuthProfileFailoverFailure = buildAuthProfileFailoverFailureText(err) !== null; const providerRequestError = !isBilling && !oauthRefreshFailure && !hasAuthProfileFailoverFailure && !params.shouldSurfaceToControlUi ? failoverFacts.providerRequestError : undefined; const restartLifecycleError = resolveRestartLifecycleError(err); if ( restartLifecycleError instanceof GatewayDrainingError || restartLifecycleError instanceof CommandLaneClearedError ) { takePendingLifecycleTerminal().emit("error", restartLifecycleError); turn.replyOperation?.fail( restartLifecycleError instanceof GatewayDrainingError ? "gateway_draining" : "command_lane_cleared", restartLifecycleError, ); return { kind: "final", payload: markAgentRunFailureReplyPayload({ text: buildRestartLifecycleReplyText() }), }; } if (isCompactionFailure) { takePendingLifecycleTerminal().emit("error", err); defaultRuntime.error( `Auto-compaction failed (${message}). Preserving existing session mapping for ${turn.sessionKey ?? turn.followupRun.run.sessionId}.`, ); turn.replyOperation?.fail("run_failed", err); return { kind: "final", payload: markAgentRunFailureReplyPayload({ text: buildContextOverflowRecoveryText({ duringCompaction: true, preserveSessionMapping: true, cfg: params.runtimeConfig, agentId: turn.followupRun.run.agentId, primaryProvider: turn.followupRun.run.provider, primaryModel: turn.followupRun.run.model, runtimeProvider: params.state.attemptedRuntimeProvider, runtimeModel: params.state.attemptedRuntimeModel, activeSessionEntry: turn.getActiveSessionEntry(), }), }), }; } const replayPrevented = findCliTimeoutError(err)?.cliTimeout.observedActivity === true; if (providerRequestError) { return await settleFailure({ // Curated facet copy beats the generic classified summary; see // buildExternalRunFailureReply for the same priority. text: providerRequestError.userMessage, }); } defaultRuntime.error(`Embedded agent failed before reply: ${message}`); const externalRunFailureCandidate = !failureSummary && !isContextOverflow ? buildExternalRunFailureReply( { message, error: err }, { includeAuthProfileId: !isNonDirectConversationContext(turn.sessionCtx), includeDetails: isVerboseFailureDetailEnabled(turn.resolvedVerboseLevel), isHeartbeat: turn.isHeartbeat, replayPrevented, failoverFacts, }, ) : undefined; const externalRunFailureReply = !params.shouldSurfaceToControlUi || externalRunFailureCandidate?.presentation || renderFailoverCodeUserCopy(failoverFacts.code) ? externalRunFailureCandidate : undefined; const fallbackText = failureSummary?.text ?? (isContextOverflow ? "⚠️ Context overflow — prompt too large for this model. Try a shorter message or a larger-context model." : (externalRunFailureReply?.text ?? (params.shouldSurfaceToControlUi ? renderControlUiAgentFailureCopy(message) : turn.isHeartbeat ? HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT : GENERIC_EXTERNAL_RUN_FAILURE_TEXT))); const userVisibleFallbackText = resolveExternalRunFailureTextForConversation({ text: fallbackText, visibleReplyDelivered: await turn.resolveVisibleReplyDelivery?.(), sessionCtx: turn.sessionCtx, isGenericRunnerFailure: externalRunFailureReply?.isGenericRunnerFailure ?? false, cfg: turn.followupRun.run.config, }); return await settleFailure({ text: userVisibleFallbackText, ...(externalRunFailureReply?.presentation ? { presentation: externalRunFailureReply.presentation } : {}), }); }