| import { feature } from 'bun:bundle' |
| import type { |
| Base64ImageSource, |
| ContentBlockParam, |
| ImageBlockParam, |
| } from '@anthropic-ai/sdk/resources/messages.mjs' |
| import { randomUUID } from 'crypto' |
| import type { QuerySource } from 'src/constants/querySource.js' |
| import { logEvent } from 'src/services/analytics/index.js' |
| import { getContentText } from 'src/utils/messages.js' |
| import { |
| findCommand, |
| getCommandName, |
| isBridgeSafeCommand, |
| type LocalJSXCommandContext, |
| } from '../../commands.js' |
| import type { CanUseToolFn } from '../../hooks/useCanUseTool.js' |
| import type { IDESelection } from '../../hooks/useIdeSelection.js' |
| import type { SetToolJSXFn, ToolUseContext } from '../../Tool.js' |
| import type { |
| AssistantMessage, |
| AttachmentMessage, |
| Message, |
| ProgressMessage, |
| SystemMessage, |
| UserMessage, |
| } from '../../types/message.js' |
| import type { PermissionMode } from '../../types/permissions.js' |
| import { |
| isValidImagePaste, |
| type PromptInputMode, |
| } from '../../types/textInputTypes.js' |
| import { |
| type AgentMentionAttachment, |
| createAttachmentMessage, |
| getAttachmentMessages, |
| } from '../attachments.js' |
| import type { PastedContent } from '../config.js' |
| import type { EffortValue } from '../effort.js' |
| import { toArray } from '../generators.js' |
| import { |
| executeUserPromptSubmitHooks, |
| getUserPromptSubmitHookBlockingMessage, |
| } from '../hooks.js' |
| import { |
| createImageMetadataText, |
| maybeResizeAndDownsampleImageBlock, |
| } from '../imageResizer.js' |
| import { storeImages } from '../imageStore.js' |
| import { |
| createCommandInputMessage, |
| createSystemMessage, |
| createUserMessage, |
| } from '../messages.js' |
| import { queryCheckpoint } from '../queryProfiler.js' |
| import { parseSlashCommand } from '../slashCommandParsing.js' |
| import { |
| hasUltraplanKeyword, |
| replaceUltraplanKeyword, |
| } from '../ultraplan/keyword.js' |
| import { processTextPrompt } from './processTextPrompt.js' |
| export type ProcessUserInputContext = ToolUseContext & LocalJSXCommandContext |
|
|
| export type ProcessUserInputBaseResult = { |
| messages: ( |
| | UserMessage |
| | AssistantMessage |
| | AttachmentMessage |
| | SystemMessage |
| | ProgressMessage |
| )[] |
| shouldQuery: boolean |
| allowedTools?: string[] |
| model?: string |
| effort?: EffortValue |
| |
| |
| resultText?: string |
| |
| |
| nextInput?: string |
| submitNextInput?: boolean |
| } |
|
|
| export async function processUserInput({ |
| input, |
| preExpansionInput, |
| mode, |
| setToolJSX, |
| context, |
| pastedContents, |
| ideSelection, |
| messages, |
| setUserInputOnProcessing, |
| uuid, |
| isAlreadyProcessing, |
| querySource, |
| canUseTool, |
| skipSlashCommands, |
| bridgeOrigin, |
| isMeta, |
| skipAttachments, |
| }: { |
| input: string | Array<ContentBlockParam> |
| /** |
| * Input before [Pasted text #N] expansion. Used for ultraplan keyword |
| * detection so pasted content containing the word cannot trigger. Falls |
| * back to the string `input` when unset. |
| */ |
| preExpansionInput?: string |
| mode: PromptInputMode |
| setToolJSX: SetToolJSXFn |
| context: ProcessUserInputContext |
| pastedContents?: Record<number, PastedContent> |
| ideSelection?: IDESelection |
| messages?: Message[] |
| setUserInputOnProcessing?: (prompt?: string) => void |
| uuid?: string |
| isAlreadyProcessing?: boolean |
| querySource?: QuerySource |
| canUseTool?: CanUseToolFn |
| /** |
| * When true, input starting with `/` is treated as plain text. |
| * Used for remotely-received messages (bridge/CCR) that should not |
| * trigger local slash commands or skills. |
| */ |
| skipSlashCommands?: boolean |
| /** |
| * When true, slash commands matching isBridgeSafeCommand() execute even |
| * though skipSlashCommands is set. See QueuedCommand.bridgeOrigin. |
| */ |
| bridgeOrigin?: boolean |
| /** |
| * When true, the resulting UserMessage gets `isMeta: true` (user-hidden, |
| * model-visible). Propagated from `QueuedCommand.isMeta` for queued |
| * system-generated prompts. |
| */ |
| isMeta?: boolean |
| skipAttachments?: boolean |
| }): Promise<ProcessUserInputBaseResult> { |
| const inputString = typeof input === 'string' ? input : null |
| |
| |
| |
| if (mode === 'prompt' && inputString !== null && !isMeta) { |
| setUserInputOnProcessing?.(inputString) |
| } |
|
|
| queryCheckpoint('query_process_user_input_base_start') |
|
|
| const appState = context.getAppState() |
|
|
| const result = await processUserInputBase( |
| input, |
| mode, |
| setToolJSX, |
| context, |
| pastedContents, |
| ideSelection, |
| messages, |
| uuid, |
| isAlreadyProcessing, |
| querySource, |
| canUseTool, |
| appState.toolPermissionContext.mode, |
| skipSlashCommands, |
| bridgeOrigin, |
| isMeta, |
| skipAttachments, |
| preExpansionInput, |
| ) |
| queryCheckpoint('query_process_user_input_base_end') |
|
|
| if (!result.shouldQuery) { |
| return result |
| } |
|
|
| |
| queryCheckpoint('query_hooks_start') |
| const inputMessage = getContentText(input) || '' |
|
|
| for await (const hookResult of executeUserPromptSubmitHooks( |
| inputMessage, |
| appState.toolPermissionContext.mode, |
| context, |
| context.requestPrompt, |
| )) { |
| |
| if (hookResult.message?.type === 'progress') { |
| continue |
| } |
|
|
| |
| if (hookResult.blockingError) { |
| const blockingMessage = getUserPromptSubmitHookBlockingMessage( |
| hookResult.blockingError, |
| ) |
| return { |
| messages: [ |
| |
| createSystemMessage( |
| `${blockingMessage}\n\nOriginal prompt: ${input}`, |
| 'warning', |
| ), |
| ], |
| shouldQuery: false, |
| allowedTools: result.allowedTools, |
| } |
| } |
|
|
| |
| |
| if (hookResult.preventContinuation) { |
| const message = hookResult.stopReason |
| ? `Operation stopped by hook: ${hookResult.stopReason}` |
| : 'Operation stopped by hook' |
| result.messages.push( |
| createUserMessage({ |
| content: message, |
| }), |
| ) |
| result.shouldQuery = false |
| return result |
| } |
|
|
| |
| if ( |
| hookResult.additionalContexts && |
| hookResult.additionalContexts.length > 0 |
| ) { |
| result.messages.push( |
| createAttachmentMessage({ |
| type: 'hook_additional_context', |
| content: hookResult.additionalContexts.map(applyTruncation), |
| hookName: 'UserPromptSubmit', |
| toolUseID: `hook-${randomUUID()}`, |
| hookEvent: 'UserPromptSubmit', |
| }), |
| ) |
| } |
|
|
| |
| if (hookResult.message) { |
| switch (hookResult.message.attachment.type) { |
| case 'hook_success': |
| if (!hookResult.message.attachment.content) { |
| |
| break |
| } |
| result.messages.push({ |
| ...hookResult.message, |
| attachment: { |
| ...hookResult.message.attachment, |
| content: applyTruncation(hookResult.message.attachment.content), |
| }, |
| }) |
| break |
| default: |
| result.messages.push(hookResult.message) |
| break |
| } |
| } |
| } |
| queryCheckpoint('query_hooks_end') |
|
|
| |
| |
| |
| return result |
| } |
|
|
| const MAX_HOOK_OUTPUT_LENGTH = 10000 |
|
|
| function applyTruncation(content: string): string { |
| if (content.length > MAX_HOOK_OUTPUT_LENGTH) { |
| return `${content.substring(0, MAX_HOOK_OUTPUT_LENGTH)}… [output truncated - exceeded ${MAX_HOOK_OUTPUT_LENGTH} characters]` |
| } |
| return content |
| } |
|
|
| async function processUserInputBase( |
| input: string | Array<ContentBlockParam>, |
| mode: PromptInputMode, |
| setToolJSX: SetToolJSXFn, |
| context: ProcessUserInputContext, |
| pastedContents?: Record<number, PastedContent>, |
| ideSelection?: IDESelection, |
| messages?: Message[], |
| uuid?: string, |
| isAlreadyProcessing?: boolean, |
| querySource?: QuerySource, |
| canUseTool?: CanUseToolFn, |
| permissionMode?: PermissionMode, |
| skipSlashCommands?: boolean, |
| bridgeOrigin?: boolean, |
| isMeta?: boolean, |
| skipAttachments?: boolean, |
| preExpansionInput?: string, |
| ): Promise<ProcessUserInputBaseResult> { |
| let inputString: string | null = null |
| let precedingInputBlocks: ContentBlockParam[] = [] |
|
|
| |
| const imageMetadataTexts: string[] = [] |
|
|
| |
| |
| |
| |
| |
| |
| let normalizedInput: string | ContentBlockParam[] = input |
|
|
| if (typeof input === 'string') { |
| inputString = input |
| } else if (input.length > 0) { |
| queryCheckpoint('query_image_processing_start') |
| const processedBlocks: ContentBlockParam[] = [] |
| for (const block of input) { |
| if (block.type === 'image') { |
| const resized = await maybeResizeAndDownsampleImageBlock(block) |
| |
| if (resized.dimensions) { |
| const metadataText = createImageMetadataText(resized.dimensions) |
| if (metadataText) { |
| imageMetadataTexts.push(metadataText) |
| } |
| } |
| processedBlocks.push(resized.block) |
| } else { |
| processedBlocks.push(block) |
| } |
| } |
| normalizedInput = processedBlocks |
| queryCheckpoint('query_image_processing_end') |
| |
| |
| const lastBlock = processedBlocks[processedBlocks.length - 1] |
| if (lastBlock?.type === 'text') { |
| inputString = lastBlock.text |
| precedingInputBlocks = processedBlocks.slice(0, -1) |
| } else { |
| precedingInputBlocks = processedBlocks |
| } |
| } |
|
|
| if (inputString === null && mode !== 'prompt') { |
| throw new Error(`Mode: ${mode} requires a string input.`) |
| } |
|
|
| |
| |
| const imageContents = pastedContents |
| ? Object.values(pastedContents).filter(isValidImagePaste) |
| : [] |
| const imagePasteIds = imageContents.map(img => img.id) |
|
|
| |
| |
| const storedImagePaths = pastedContents |
| ? await storeImages(pastedContents) |
| : new Map<number, string>() |
|
|
| |
| queryCheckpoint('query_pasted_image_processing_start') |
| const imageProcessingResults = await Promise.all( |
| imageContents.map(async pastedImage => { |
| const imageBlock: ImageBlockParam = { |
| type: 'image', |
| source: { |
| type: 'base64', |
| media_type: (pastedImage.mediaType || |
| 'image/png') as Base64ImageSource['media_type'], |
| data: pastedImage.content, |
| }, |
| } |
| logEvent('tengu_pasted_image_resize_attempt', { |
| original_size_bytes: pastedImage.content.length, |
| }) |
| const resized = await maybeResizeAndDownsampleImageBlock(imageBlock) |
| return { |
| resized, |
| originalDimensions: pastedImage.dimensions, |
| sourcePath: |
| pastedImage.sourcePath ?? storedImagePaths.get(pastedImage.id), |
| } |
| }), |
| ) |
| |
| const imageContentBlocks: ContentBlockParam[] = [] |
| for (const { |
| resized, |
| originalDimensions, |
| sourcePath, |
| } of imageProcessingResults) { |
| |
| if (resized.dimensions) { |
| const metadataText = createImageMetadataText( |
| resized.dimensions, |
| sourcePath, |
| ) |
| if (metadataText) { |
| imageMetadataTexts.push(metadataText) |
| } |
| } else if (originalDimensions) { |
| |
| const metadataText = createImageMetadataText( |
| originalDimensions, |
| sourcePath, |
| ) |
| if (metadataText) { |
| imageMetadataTexts.push(metadataText) |
| } |
| } else if (sourcePath) { |
| |
| imageMetadataTexts.push(`[Image source: ${sourcePath}]`) |
| } |
| imageContentBlocks.push(resized.block) |
| } |
| queryCheckpoint('query_pasted_image_processing_end') |
|
|
| |
| |
| |
| |
| |
| |
| let effectiveSkipSlash = skipSlashCommands |
| if (bridgeOrigin && inputString !== null && inputString.startsWith('/')) { |
| const parsed = parseSlashCommand(inputString) |
| const cmd = parsed |
| ? findCommand(parsed.commandName, context.options.commands) |
| : undefined |
| if (cmd) { |
| if (isBridgeSafeCommand(cmd)) { |
| effectiveSkipSlash = false |
| } else { |
| const msg = `/${getCommandName(cmd)} isn't available over Remote Control.` |
| return { |
| messages: [ |
| createUserMessage({ content: inputString, uuid }), |
| createCommandInputMessage( |
| `<local-command-stdout>${msg}</local-command-stdout>`, |
| ), |
| ], |
| shouldQuery: false, |
| resultText: msg, |
| } |
| } |
| } |
| |
| |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| if ( |
| feature('ULTRAPLAN') && |
| mode === 'prompt' && |
| !context.options.isNonInteractiveSession && |
| inputString !== null && |
| !effectiveSkipSlash && |
| !inputString.startsWith('/') && |
| !context.getAppState().ultraplanSessionUrl && |
| !context.getAppState().ultraplanLaunching && |
| hasUltraplanKeyword(preExpansionInput ?? inputString) |
| ) { |
| logEvent('tengu_ultraplan_keyword', {}) |
| const rewritten = replaceUltraplanKeyword(inputString).trim() |
| const { processSlashCommand } = await import('./processSlashCommand.js') |
| const slashResult = await processSlashCommand( |
| `/ultraplan ${rewritten}`, |
| precedingInputBlocks, |
| imageContentBlocks, |
| [], |
| context, |
| setToolJSX, |
| uuid, |
| isAlreadyProcessing, |
| canUseTool, |
| ) |
| return addImageMetadataMessage(slashResult, imageMetadataTexts) |
| } |
|
|
| |
| const shouldExtractAttachments = |
| !skipAttachments && |
| inputString !== null && |
| (mode !== 'prompt' || effectiveSkipSlash || !inputString.startsWith('/')) |
|
|
| queryCheckpoint('query_attachment_loading_start') |
| const attachmentMessages = shouldExtractAttachments |
| ? await toArray( |
| getAttachmentMessages( |
| inputString, |
| context, |
| ideSelection ?? null, |
| [], |
| messages, |
| querySource, |
| ), |
| ) |
| : [] |
| queryCheckpoint('query_attachment_loading_end') |
|
|
| |
| if (inputString !== null && mode === 'bash') { |
| const { processBashCommand } = await import('./processBashCommand.js') |
| return addImageMetadataMessage( |
| await processBashCommand( |
| inputString, |
| precedingInputBlocks, |
| attachmentMessages, |
| context, |
| setToolJSX, |
| ), |
| imageMetadataTexts, |
| ) |
| } |
|
|
| |
| |
| if ( |
| inputString !== null && |
| !effectiveSkipSlash && |
| inputString.startsWith('/') |
| ) { |
| const { processSlashCommand } = await import('./processSlashCommand.js') |
| const slashResult = await processSlashCommand( |
| inputString, |
| precedingInputBlocks, |
| imageContentBlocks, |
| attachmentMessages, |
| context, |
| setToolJSX, |
| uuid, |
| isAlreadyProcessing, |
| canUseTool, |
| ) |
| return addImageMetadataMessage(slashResult, imageMetadataTexts) |
| } |
|
|
| |
| if (inputString !== null && mode === 'prompt') { |
| const trimmedInput = inputString.trim() |
|
|
| const agentMention = attachmentMessages.find( |
| (m): m is AttachmentMessage<AgentMentionAttachment> => |
| m.attachment.type === 'agent_mention', |
| ) |
|
|
| if (agentMention) { |
| const agentMentionString = `@agent-${agentMention.attachment.agentType}` |
| const isSubagentOnly = trimmedInput === agentMentionString |
| const isPrefix = |
| trimmedInput.startsWith(agentMentionString) && !isSubagentOnly |
|
|
| |
| logEvent('tengu_subagent_at_mention', { |
| is_subagent_only: isSubagentOnly, |
| is_prefix: isPrefix, |
| }) |
| } |
| } |
|
|
| |
| return addImageMetadataMessage( |
| processTextPrompt( |
| normalizedInput, |
| imageContentBlocks, |
| imagePasteIds, |
| attachmentMessages, |
| uuid, |
| permissionMode, |
| isMeta, |
| ), |
| imageMetadataTexts, |
| ) |
| } |
|
|
| |
| function addImageMetadataMessage( |
| result: ProcessUserInputBaseResult, |
| imageMetadataTexts: string[], |
| ): ProcessUserInputBaseResult { |
| if (imageMetadataTexts.length > 0) { |
| result.messages.push( |
| createUserMessage({ |
| content: imageMetadataTexts.map(text => ({ type: 'text', text })), |
| isMeta: true, |
| }), |
| ) |
| } |
| return result |
| } |
|
|