import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers'; import { formatAttachmentText } from '$lib/utils/formatters'; import { isAbortError } from '$lib/utils/abort'; import { streamIdentity } from '$lib/utils/stream-identity'; import { ATTACHMENT_LABEL_PDF_FILE, ATTACHMENT_LABEL_MCP_PROMPT, ATTACHMENT_LABEL_MCP_RESOURCE, LEGACY_AGENTIC_REGEX, REASONING_EFFORT_TOKENS, SETTINGS_KEYS, API_CHAT, API_SLOTS, CONTROL_ACTION, SSE_LINE_SEPARATOR, SSE_DATA_PREFIX, SSE_DONE_MARKER, STREAM_VISIBILITY_KICK_MS, STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX, API_STREAM } from '$lib/constants'; import { AttachmentType, ContentPartType, FileTypeAudio, MessageRole, MimeTypeAudio, ReasoningFormat, StreamConnectionState } from '$lib/enums'; import type { ApiChatMessageContentPart, ApiChatMessageData, ApiChatCompletionToolCall, ApiStreamSession } from '$lib/types/api'; import type { AudioInputFormat, DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types'; import { modelsStore } from '$lib/stores/models.svelte'; import { settingsStore } from '../stores/settings.svelte'; import { capImageDataURLSize } from '../utils/cap-img-size'; function getAudioInputFormat(mimeType: string): AudioInputFormat { const normalizedMimeType = mimeType.trim().toLowerCase(); if ( normalizedMimeType === MimeTypeAudio.WAV || normalizedMimeType === MimeTypeAudio.WAVE || normalizedMimeType === MimeTypeAudio.X_WAV || normalizedMimeType === MimeTypeAudio.X_WAVE || normalizedMimeType === MimeTypeAudio.VND_WAVE || normalizedMimeType === MimeTypeAudio.X_PN_WAV ) { return FileTypeAudio.WAV; } return FileTypeAudio.MP3; } interface ResumableStreamState { bytesReceived: number; updatedAt: number; // model frozen at POST time, lets a reload rebuild the exact conv::model identity the // server keyed the session under. null when the POST carried no explicit model model?: string | null; } function streamStorageKey(conversationId: string): string { return STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX + conversationId; } export class ChatService { /** * * * Title Generation * * */ /** * Sends a streaming chat completion request for generating a chat title. * Delegates to `sendMessage` for fetch, SSE parsing, and error handling. * * @param message - The single message to send (a user message containing the title generation prompt) * @param model - Optional model name to use (required in ROUTER mode) * @param signal - Optional AbortSignal to cancel the request * @returns {Promise} The aggregated title text, or empty string if request failed * @static */ static async generateTitle( message: ApiChatMessageData, model?: string | null, signal?: AbortSignal ): Promise { let titleResponse = ''; try { await ChatService.sendMessage( [message], { model: model || undefined, stream: true, custom: { chat_template_kwargs: { enable_thinking: false } }, onChunk: (chunk: string) => { titleResponse += chunk; } }, undefined, signal ); } catch { return ''; } return titleResponse; } /** * * * Messaging * * */ /** * Sends a chat completion request to the llama-server. * Supports both streaming and non-streaming responses with comprehensive parameter configuration. * Automatically converts database messages with attachments to the appropriate API format. * * @param messages - Array of chat messages to send to the API (supports both ApiChatMessageData and DatabaseMessage with attachments) * @param options - Configuration options for the chat completion request. See `SettingsChatServiceOptions` type for details. * @returns {Promise} that resolves to the complete response string (non-streaming) or void (streaming) * @throws {Error} if the request fails or is aborted */ static async sendMessage( messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], options: SettingsChatServiceOptions = {}, conversationId?: string, signal?: AbortSignal ): Promise { const { stream, onChunk, onComplete, onError, onConnectionState, onReasoningChunk, onToolCallChunk, onModel, onCompletionId, onTimings, // Tools for function calling tools, // Generation parameters temperature, max_tokens, // Sampling parameters dynatemp_range, dynatemp_exponent, top_k, top_p, min_p, xtc_probability, xtc_threshold, typ_p, // Penalty parameters repeat_last_n, repeat_penalty, presence_penalty, frequency_penalty, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, // Other parameters samplers, backend_sampling, custom, timings_per_token, // Config options disableReasoningParsing, excludeReasoningFromContext, enableThinking, reasoningEffort, continueFinalMessage } = options; const normalizedMessages: ApiChatMessageData[] = ( await Promise.all( messages.map((msg) => { if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { const dbMsg = msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }; return ChatService.convertDbMessageToApiChatMessageData(dbMsg); } else { return msg as ApiChatMessageData; } }) ) ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { // Filter out empty system messages if (msg.role === MessageRole.SYSTEM) { const content = typeof msg.content === 'string' ? msg.content : ''; return content.trim().length > 0; } return true; }); // Filter out image attachments if the model doesn't support vision if (options.model && !modelsStore.modelSupportsVision(options.model)) { normalizedMessages.forEach((msg) => { if (Array.isArray(msg.content)) { msg.content = msg.content.filter((part: ApiChatMessageContentPart) => { if (part.type === ContentPartType.IMAGE_URL) { console.info( `[ChatService] Skipping image attachment in message history (model "${options.model}" does not support vision)` ); return false; } return true; }); // If only text remains and it's a single part, simplify to string if ( msg.content.length === 1 && msg.content[0].type === ContentPartType.TEXT && typeof msg.content[0].text === 'string' ) { msg.content = msg.content[0].text; } } }); } const requestBody: ApiChatCompletionRequest = { messages: normalizedMessages.map((msg: ApiChatMessageData) => { const mapped: ApiChatCompletionRequest['messages'][0] = { role: msg.role, content: msg.content, tool_calls: msg.tool_calls, tool_call_id: msg.tool_call_id }; // Include reasoning_content from the dedicated field if (!excludeReasoningFromContext && msg.reasoning_content) { mapped.reasoning_content = msg.reasoning_content; } return mapped; }), stream, return_progress: stream ? true : undefined, sse_ping_interval: stream ? 1 : undefined, tools: tools && tools.length > 0 ? tools : undefined }; // Include model in request if provided (required in ROUTER mode) if (options.model) { requestBody.model = options.model; } requestBody.reasoning_format = disableReasoningParsing ? ReasoningFormat.NONE : ReasoningFormat.AUTO; const reasoningBudgetTokens = enableThinking && reasoningEffort ? (REASONING_EFFORT_TOKENS[reasoningEffort] ?? -1) : -1; requestBody.chat_template_kwargs = { ...(requestBody.chat_template_kwargs ?? {}), enable_thinking: enableThinking }; if (reasoningBudgetTokens >= 0) { requestBody.thinking_budget_tokens = reasoningBudgetTokens; } // arms the budget sampler so reasoning can be ended at runtime via the control endpoint requestBody.reasoning_control = true; if (continueFinalMessage) { requestBody.continue_final_message = true; requestBody.add_generation_prompt = false; } if (temperature !== undefined) requestBody.temperature = temperature; if (max_tokens !== undefined) { // Set max_tokens to -1 (infinite) when explicitly configured as 0 or null requestBody.max_tokens = max_tokens !== null && max_tokens !== 0 ? max_tokens : -1; } if (dynatemp_range !== undefined) requestBody.dynatemp_range = dynatemp_range; if (dynatemp_exponent !== undefined) requestBody.dynatemp_exponent = dynatemp_exponent; if (top_k !== undefined) requestBody.top_k = top_k; if (top_p !== undefined) requestBody.top_p = top_p; if (min_p !== undefined) requestBody.min_p = min_p; if (xtc_probability !== undefined) requestBody.xtc_probability = xtc_probability; if (xtc_threshold !== undefined) requestBody.xtc_threshold = xtc_threshold; if (typ_p !== undefined) requestBody.typ_p = typ_p; if (repeat_last_n !== undefined) requestBody.repeat_last_n = repeat_last_n; if (repeat_penalty !== undefined) requestBody.repeat_penalty = repeat_penalty; if (presence_penalty !== undefined) requestBody.presence_penalty = presence_penalty; if (frequency_penalty !== undefined) requestBody.frequency_penalty = frequency_penalty; if (dry_multiplier !== undefined) requestBody.dry_multiplier = dry_multiplier; if (dry_base !== undefined) requestBody.dry_base = dry_base; if (dry_allowed_length !== undefined) requestBody.dry_allowed_length = dry_allowed_length; if (dry_penalty_last_n !== undefined) requestBody.dry_penalty_last_n = dry_penalty_last_n; if (samplers !== undefined) { requestBody.samplers = typeof samplers === 'string' ? samplers.split(';').filter((s: string) => s.trim()) : samplers; } if (backend_sampling !== undefined) requestBody.backend_sampling = backend_sampling; if (timings_per_token !== undefined) requestBody.timings_per_token = timings_per_token; if (custom) { try { const customParams = typeof custom === 'string' ? JSON.parse(custom) : custom; Object.assign(requestBody, customParams); } catch (error) { console.warn('Failed to parse custom parameters:', error); } } try { const headers: Record = { ...getJsonHeaders() }; // tag streaming requests with the conversation id, this single header is the opt in for the // server side replay buffer and powers discoverActiveStream on tab reopen. with an explicit // model the ::model suffix keeps the per model session distinct if (stream && conversationId) { headers['X-Conversation-Id'] = streamIdentity(conversationId, options.model); } const response = await fetch(API_CHAT.COMPLETIONS, { method: 'POST', headers, body: JSON.stringify(requestBody), signal }); if (!response.ok) { const error = await ChatService.parseErrorResponse(response); if (onError) { onError(error); } throw error; } if (stream) { await ChatService.handleStreamResponse( response, onChunk, onComplete, onError, onReasoningChunk, onToolCallChunk, onModel, onCompletionId, onTimings, conversationId, signal, onConnectionState, options.model ); return; } else { return ChatService.handleNonStreamResponse( response, onComplete, onError, onToolCallChunk, onModel ); } } catch (error) { if (isAbortError(error)) { console.log('Chat completion request was aborted'); return; } let userFriendlyError: Error; if (error instanceof Error) { if (error.name === 'TypeError' && error.message.includes('fetch')) { userFriendlyError = new Error( 'Unable to connect to server - please check if the server is running' ); userFriendlyError.name = 'NetworkError'; } else if (error.message.includes('ECONNREFUSED')) { userFriendlyError = new Error('Connection refused - server may be offline'); userFriendlyError.name = 'NetworkError'; } else if (error.message.includes('ETIMEDOUT')) { userFriendlyError = new Error('Request timed out - the server took too long to respond'); userFriendlyError.name = 'TimeoutError'; } else { userFriendlyError = error; } } else { userFriendlyError = new Error('Unknown error occurred while sending message'); } console.error('Error in sendMessage:', error); if (onError) { onError(userFriendlyError); } throw userFriendlyError; } } /** * Checks whether all server slots are currently idle (not processing any requests). * Queries the /slots endpoint (requires --slots flag on the server). * Returns true if all slots are idle, false if any is processing. * If the endpoint is unavailable or errors out, returns true (best-effort fallback). * * @param signal - Optional AbortSignal to cancel the request if needed * @param model - Optional model name to check slots for (required in ROUTER mode) * @returns {Promise} Promise that resolves to true if all slots are idle, false if any is processing */ static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise { try { const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST; const res = await fetch(url, { signal }); if (!res.ok) return true; const slots: { is_processing: boolean }[] = await res.json(); return slots.every((s) => !s.is_processing); } catch { return true; } } /** * Ends the current reasoning block of a running completion, targeted by its * chat completion id (streamed back as `id`). Matching the completion rather * than a slot index avoids a TOCTOU: a finished completion simply matches * nothing server side. The model is carried so the router forwards to the * right child, single model ignores it. Returns true on success. */ static async stopReasoning(completionId: string, model?: string | null): Promise { if (!completionId) { console.error( 'stopReasoning: no completion id for the active message, cannot target the running completion' ); return false; } const body: Record = { id: completionId, action: CONTROL_ACTION.END_REASONING }; if (model) body.model = model; try { const res = await fetch(API_CHAT.CONTROL, { method: 'POST', headers: getJsonHeaders(), body: JSON.stringify(body) }); const data = await res.json().catch(() => null); if (!res.ok || data?.success !== true) { console.error('stopReasoning: control request failed', { status: res.status, completionId, response: data }); return false; } return true; } catch (error) { console.error('stopReasoning: control request threw', { completionId, error }); return false; } } /** * Sends a fire-and-forget request to pre-encode the conversation in the server's KV cache. * After a response completes, this re-submits the full conversation * using n_predict=0 and stream=false so the server processes the prompt without generating tokens. * This warms the cache for the next turn, making it faster. * * When excludeReasoningFromContext is true, reasoning content is stripped from the messages * to match what sendMessage would send on the next turn (avoiding cache misses). * When false, reasoning_content is preserved so the cached prompt matches the next request. * * @param messages - The full conversation including the latest assistant response * @param model - Optional model name (required in ROUTER mode) * @param excludeReasoning - Whether to strip reasoning content (should match excludeReasoningFromContext setting) * @param signal - Optional AbortSignal to cancel the pre-encode request */ static async cancelServerStream(conversationId: string, model?: string | null): Promise { if (!conversationId) return; try { const id = streamIdentity(conversationId, model); await fetch(`${API_STREAM.BASE}/${encodeURIComponent(id)}`, { method: 'DELETE', headers: getAuthHeaders() }); } catch (e) { console.warn('cancelServerStream failed:', e); } } /** * Pick the running session to splice into when discoverActiveStream lists candidates for a * conversation. Finalized sessions are not candidates: their final content was already written * to the DB by the original onComplete handler, so attaching to them would replay a buffer that * may not match what the DB holds. A continue session's buffer holds only the appended deltas, * not the pre continue prefix, so replaying it as a fresh generation would erase the original. * * Among running sessions we tie break on the most recent started_at, which covers the case of * multiple inferences left running on the same conversation. */ static selectActiveStream( sessions: ApiStreamSession[] | null | undefined ): ApiStreamSession | null { if (!Array.isArray(sessions) || sessions.length === 0) { return null; } const running = sessions.filter((s) => !s.is_done); if (running.length === 0) { return null; } return running.reduce((best, cur) => (cur.started_at > best.started_at ? cur : best)); } // persist the running byte count and the frozen model for a conversation, a later visit // resumes the SSE replay at the right offset under the same conv::model identity static saveStreamState( conversationId: string, bytesReceived: number, model?: string | null ): void { if (!conversationId) return; try { const state: ResumableStreamState = { bytesReceived, updatedAt: Date.now(), model: model ?? null }; localStorage.setItem(streamStorageKey(conversationId), JSON.stringify(state)); } catch { // localStorage may be full or disabled, silently ignore } } static getStreamState(conversationId: string): ResumableStreamState | null { if (!conversationId) return null; try { const raw = localStorage.getItem(streamStorageKey(conversationId)); if (!raw) return null; const parsed = JSON.parse(raw) as ResumableStreamState; if (!parsed || typeof parsed.bytesReceived !== 'number') return null; return parsed; } catch { return null; } } static clearStreamState(conversationId: string): void { if (!conversationId) return; try { localStorage.removeItem(streamStorageKey(conversationId)); } catch { // nothing to do } } /** * Rebuild the stream identity for a resume. The model persisted at POST time wins, including a * stored null which means the POST carried no explicit model so the identity stays the bare conv * id. Only fall back to the caller supplied current model when nothing was persisted. */ static resumeStreamIdentity( conversationId: string, state: ResumableStreamState | null, fallbackModel: string | null ): string { const model = state && state.model !== undefined ? state.model : fallbackModel; return streamIdentity(conversationId, model); } /** * Reconnect to an interrupted stream for this conversation. Returns the fetch Response so the * existing SSE parser drains it like a fresh stream. The server returns 200 on success, 404 if * no session exists for the conv_id, and 400 if the offset is below the dropped prefix. */ static async resumeStream( conversationId: string, signal?: AbortSignal, model?: string | null ): Promise { if (!conversationId) return null; const state = ChatService.getStreamState(conversationId); const from = state?.bytesReceived ?? 0; const id = streamIdentity(conversationId, model); const url = `${API_STREAM.BASE}/${encodeURIComponent(id)}?from=${from}`; return await fetch(url, { method: 'GET', signal, headers: getAuthHeaders() }); } static async preEncode( messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], model?: string | null, excludeReasoning?: boolean, signal?: AbortSignal ): Promise { const normalizedMessages: ApiChatMessageData[] = ( await Promise.all( messages.map((msg) => { if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { return ChatService.convertDbMessageToApiChatMessageData( msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] } ); } return msg as ApiChatMessageData; }) ) ).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => { if (msg.role === MessageRole.SYSTEM) { const content = typeof msg.content === 'string' ? msg.content : ''; return content.trim().length > 0; } return true; }); const requestBody: Record = { messages: normalizedMessages.map((msg: ApiChatMessageData) => { const mapped: Record = { role: msg.role, content: excludeReasoning ? ChatService.stripReasoningContent(msg.content) : msg.content, tool_calls: msg.tool_calls, tool_call_id: msg.tool_call_id }; if (!excludeReasoning && msg.reasoning_content) { mapped.reasoning_content = msg.reasoning_content; } return mapped; }), stream: false, n_predict: 0 }; if (model) { requestBody.model = model; } try { await fetch(API_CHAT.COMPLETIONS, { method: 'POST', headers: getJsonHeaders(), body: JSON.stringify(requestBody), signal }); } catch (error) { if (!isAbortError(error)) { console.warn('[ChatService] Pre-encode request failed:', error); } } } /** * * * Streaming * * */ /** * Handles streaming response from the chat completion API * @param response - The Response object from the fetch request * @param onChunk - Optional callback invoked for each content chunk received * @param onComplete - Optional callback invoked when the stream is complete with full response * @param onError - Optional callback invoked if an error occurs during streaming * @param onReasoningChunk - Optional callback invoked for each reasoning content chunk * @param conversationId - Optional conversation ID for per-conversation state tracking * @returns {Promise} Promise that resolves when streaming is complete * @throws {Error} if the stream cannot be read or parsed */ static async handleStreamResponse( response: Response, onChunk?: (chunk: string) => void, onComplete?: ( response: string, reasoningContent?: string, timings?: ChatMessageTimings, toolCalls?: string ) => void, onError?: (error: Error) => void, onReasoningChunk?: (chunk: string) => void, onToolCallChunk?: (chunk: string) => void, onModel?: (model: string) => void, onCompletionId?: (id: string) => void, onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, conversationId?: string, abortSignal?: AbortSignal, onConnectionState?: (state: StreamConnectionState) => void, streamModel?: string | null ): Promise { let reader = response.body?.getReader(); if (!reader) { throw new Error('No response body'); } // bytesParsed is the absolute server side buffer offset of the next byte to parse // segmentStartOffset is the absolute offset where the current reader started, reset on resume // segmentBytesRead is wire bytes read by the current reader let bytesParsed = 0; let segmentStartOffset = 0; let segmentBytesRead = 0; let lastByteAt = Date.now(); // each resume must produce at least one byte to be retried again // if a resume returns 200 but yields nothing, we abandon // since the session has a bounded size, the total number of retries is bounded by construction let madeProgress = true; const encoder = new TextEncoder(); if (conversationId) { ChatService.saveStreamState(conversationId, 0, streamModel); } onConnectionState?.(StreamConnectionState.STREAMING); let decoder = new TextDecoder(); let aggregatedContent = ''; let fullReasoningContent = ''; let aggregatedToolCalls: ApiChatCompletionToolCall[] = []; let lastTimings: ChatMessageTimings | undefined; let streamFinished = false; let modelEmitted = false; let idEmitted = false; let toolCallIndexOffset = 0; let hasOpenToolCallBatch = false; const finalizeOpenToolCallBatch = () => { if (!hasOpenToolCallBatch) { return; } toolCallIndexOffset = aggregatedToolCalls.length; hasOpenToolCallBatch = false; }; const processToolCallDelta = (toolCalls?: ApiChatCompletionToolCallDelta[]) => { if (!toolCalls || toolCalls.length === 0) { return; } aggregatedToolCalls = ChatService.mergeToolCallDeltas( aggregatedToolCalls, toolCalls, toolCallIndexOffset ); if (aggregatedToolCalls.length === 0) { return; } hasOpenToolCallBatch = true; const serializedToolCalls = JSON.stringify(aggregatedToolCalls); if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { console.log('[ChatService] Aggregated tool calls:', serializedToolCalls); } if (!serializedToolCalls) { return; } if (!abortSignal?.aborted) { onToolCallChunk?.(serializedToolCalls); } }; const onVisibilityChange = () => { if (typeof document === 'undefined') return; if (document.visibilityState !== 'visible') return; if (streamFinished) return; if (!conversationId) return; // the bytes have been quiet for too long, the OS likely killed the socket // kicking the reader unblocks reader.read with done=true so the outer loop can resume if (Date.now() - lastByteAt > STREAM_VISIBILITY_KICK_MS) { reader!.cancel().catch(() => {}); } }; if (typeof document !== 'undefined') { document.addEventListener('visibilitychange', onVisibilityChange); } try { let chunk = ''; // outer loop drives the resume cycle, swaps reader on premature end of stream while (true) { while (true) { if (abortSignal?.aborted) break; let done: boolean; let value: Uint8Array | undefined; try { const r = await reader.read(); done = r.done; value = r.value; } catch (readErr) { // reader.read() rejects with TypeError when the underlying connection drops // instead of just resolving with done=true. treat it like done so the outer // loop swaps reader via the resume path if (isAbortError(readErr)) { throw readErr; } console.warn('reader.read() rejected, treating as premature end:', readErr); done = true; value = undefined; } if (done) break; if (abortSignal?.aborted) break; if (value && value.byteLength > 0) { segmentBytesRead += value.byteLength; lastByteAt = Date.now(); if (!madeProgress) { madeProgress = true; onConnectionState?.(StreamConnectionState.STREAMING); } } chunk += decoder.decode(value, { stream: true }); const lines = chunk.split(SSE_LINE_SEPARATOR); chunk = lines.pop() || ''; // the persisted offset must point right after the last fully parsed line, // the trailing `chunk` is partial bytes still waiting for a newline if (conversationId) { const tailBytes = encoder.encode(chunk).byteLength; bytesParsed = segmentStartOffset + segmentBytesRead - tailBytes; ChatService.saveStreamState(conversationId, bytesParsed, streamModel); } for (const line of lines) { if (abortSignal?.aborted) break; if (line.startsWith(SSE_DATA_PREFIX)) { const data = line.slice(SSE_DATA_PREFIX.length).trim(); if (data === SSE_DONE_MARKER) { streamFinished = true; continue; } try { const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); const choice = parsed.choices?.[0]; const content = choice?.delta?.content; const reasoningContent = choice?.delta?.reasoning_content; const toolCalls = choice?.delta?.tool_calls; const timings = parsed.timings; const promptProgress = parsed.prompt_progress; const chunkModel = ChatService.extractModelName(parsed); if (chunkModel && !modelEmitted) { modelEmitted = true; onModel?.(chunkModel); } if (parsed.id && !idEmitted) { idEmitted = true; onCompletionId?.(parsed.id); } if (promptProgress) { ChatService.notifyTimings(undefined, promptProgress, onTimings); } if (timings) { ChatService.notifyTimings(timings, promptProgress, onTimings); lastTimings = timings; } if (content) { finalizeOpenToolCallBatch(); aggregatedContent += content; if (!abortSignal?.aborted) { onChunk?.(content); } } if (reasoningContent) { finalizeOpenToolCallBatch(); fullReasoningContent += reasoningContent; if (!abortSignal?.aborted) { onReasoningChunk?.(reasoningContent); } } processToolCallDelta(toolCalls); } catch (e) { console.error('Error parsing JSON chunk:', e); } } } if (abortSignal?.aborted) break; if (streamFinished) break; } // inner reader done, decide whether to try a resume if (abortSignal?.aborted) break; if (streamFinished) break; if (!conversationId) break; if (!madeProgress) { onConnectionState?.(StreamConnectionState.LOST); onError?.(new Error('Stream resume produced no new bytes, giving up')); break; } onConnectionState?.(StreamConnectionState.RESUMING); madeProgress = false; // the server resends starting at bytesParsed, discard any partial line we held, it // will be retransmitted from a clean line boundary. reuse the frozen model, not the // live dropdown const resumeResp = await ChatService.resumeStream( conversationId, abortSignal, streamModel ).catch(() => null); // an abort landing during the resume request is intentional, not a lost connection if (abortSignal?.aborted) break; if (!resumeResp || resumeResp.status !== 200) { onConnectionState?.(StreamConnectionState.LOST); onError?.(new Error('Stream connection lost and could not be resumed')); break; } const newReader = resumeResp.body?.getReader(); if (!newReader) break; try { reader.releaseLock(); } catch { /* ignore */ } reader = newReader; decoder = new TextDecoder(); chunk = ''; segmentStartOffset = bytesParsed; segmentBytesRead = 0; lastByteAt = Date.now(); } if (abortSignal?.aborted) return; if (streamFinished) { finalizeOpenToolCallBatch(); if (conversationId) { ChatService.clearStreamState(conversationId); } const finalToolCalls = aggregatedToolCalls.length > 0 ? JSON.stringify(aggregatedToolCalls) : undefined; onComplete?.( aggregatedContent, fullReasoningContent || undefined, lastTimings, finalToolCalls ); } } catch (error) { const err = error instanceof Error ? error : new Error('Stream error'); onError?.(err); throw err; } finally { if (typeof document !== 'undefined') { document.removeEventListener('visibilitychange', onVisibilityChange); } try { reader.releaseLock(); } catch { /* ignore */ } } } /** * Handles non-streaming response from the chat completion API. * Parses the JSON response and extracts the generated content. * * @param response - The fetch Response object containing the JSON data * @param onComplete - Optional callback invoked when response is successfully parsed * @param onError - Optional callback invoked if an error occurs during parsing * @returns {Promise} Promise that resolves to the generated content string * @throws {Error} if the response cannot be parsed or is malformed */ private static async handleNonStreamResponse( response: Response, onComplete?: ( response: string, reasoningContent?: string, timings?: ChatMessageTimings, toolCalls?: string ) => void, onError?: (error: Error) => void, onToolCallChunk?: (chunk: string) => void, onModel?: (model: string) => void ): Promise { try { const responseText = await response.text(); if (!responseText.trim()) { const noResponseError = new Error('No response received from server. Please try again.'); throw noResponseError; } const data: ApiChatCompletionResponse = JSON.parse(responseText); const responseModel = ChatService.extractModelName(data); if (responseModel) { onModel?.(responseModel); } const content = data.choices[0]?.message?.content || ''; const reasoningContent = data.choices[0]?.message?.reasoning_content; const toolCalls = data.choices[0]?.message?.tool_calls; let serializedToolCalls: string | undefined; if (toolCalls && toolCalls.length > 0) { const mergedToolCalls = ChatService.mergeToolCallDeltas([], toolCalls); if (mergedToolCalls.length > 0) { serializedToolCalls = JSON.stringify(mergedToolCalls); if (serializedToolCalls) { onToolCallChunk?.(serializedToolCalls); } } } if (!content.trim() && !serializedToolCalls) { const noResponseError = new Error('No response received from server. Please try again.'); throw noResponseError; } onComplete?.(content, reasoningContent, undefined, serializedToolCalls); return content; } catch (error) { const err = error instanceof Error ? error : new Error('Parse error'); onError?.(err); throw err; } } /** * Merges tool call deltas into an existing array of tool calls. * Handles both existing and new tool calls, updating existing ones and adding new ones. * * @param existing - The existing array of tool calls to merge into * @param deltas - The array of tool call deltas to merge * @param indexOffset - Optional offset to apply to the index of new tool calls * @returns {ApiChatCompletionToolCall[]} The merged array of tool calls */ private static mergeToolCallDeltas( existing: ApiChatCompletionToolCall[], deltas: ApiChatCompletionToolCallDelta[], indexOffset = 0 ): ApiChatCompletionToolCall[] { const result = existing.map((call) => ({ ...call, function: call.function ? { ...call.function } : undefined })); for (const delta of deltas) { const index = typeof delta.index === 'number' && delta.index >= 0 ? delta.index + indexOffset : result.length; while (result.length <= index) { result.push({ function: undefined }); } const target = result[index]!; if (delta.id) { target.id = delta.id; } if (delta.type) { target.type = delta.type; } if (delta.function) { const fn = target.function ? { ...target.function } : {}; if (delta.function.name) { fn.name = delta.function.name; } if (delta.function.arguments) { fn.arguments = (fn.arguments ?? '') + delta.function.arguments; } target.function = fn; } } return result; } /** * * * Conversion * * */ /** * Converts a database message with attachments to API chat message format. * Processes various attachment types (images, text files, PDFs) and formats them * as content parts suitable for the chat completion API. * * @param message - Database message object with optional extra attachments * @param message.content - The text content of the message * @param message.role - The role of the message sender (user, assistant, system) * @param message.extra - Optional array of message attachments (images, files, etc.) * @returns {ApiChatMessageData} object formatted for the chat completion API * @static */ static async convertDbMessageToApiChatMessageData( message: DatabaseMessage & { extra?: DatabaseMessageExtra[] } ): Promise { // Handle tool result messages (role: 'tool') if (message.role === MessageRole.TOOL && message.toolCallId) { return { role: MessageRole.TOOL, content: message.content, tool_call_id: message.toolCallId }; } // Parse tool calls for assistant messages let toolCalls: ApiChatCompletionToolCall[] | undefined; if (message.toolCalls) { try { toolCalls = JSON.parse(message.toolCalls); } catch { // Ignore parse errors for malformed tool calls } } if (!message.extra || message.extra.length === 0) { const result: ApiChatMessageData = { role: message.role as MessageRole, content: message.content }; if (message.reasoningContent) { result.reasoning_content = message.reasoningContent; } if (toolCalls && toolCalls.length > 0) { result.tool_calls = toolCalls; } return result; } const contentParts: ApiChatMessageContentPart[] = []; const textFiles = message.extra.filter( (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraTextFile => extra.type === AttachmentType.TEXT ); for (const textFile of textFiles) { contentParts.push({ type: ContentPartType.TEXT, text: formatAttachmentText('File', textFile.name, textFile.content) }); } // Handle legacy 'context' type from the old UI (pasted content) const legacyContextFiles = message.extra.filter( (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraLegacyContext => extra.type === AttachmentType.LEGACY_CONTEXT ); for (const legacyContextFile of legacyContextFiles) { contentParts.push({ type: ContentPartType.TEXT, text: formatAttachmentText('File', legacyContextFile.name, legacyContextFile.content) }); } const imageFiles = message.extra.filter( (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile => extra.type === AttachmentType.IMAGE ); for (const image of imageFiles) { const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION); // Caps the resolution and bakes the jpeg exif orientation in one pass, // untouched images pass through as is const base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution); contentParts.push({ type: ContentPartType.IMAGE_URL, image_url: { url: base64Url } }); } const audioFiles = message.extra.filter( (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraAudioFile => extra.type === AttachmentType.AUDIO ); for (const audio of audioFiles) { contentParts.push({ type: ContentPartType.INPUT_AUDIO, input_audio: { data: audio.base64Data, format: getAudioInputFormat(audio.mimeType) } }); } if (message.content) { contentParts.push({ type: ContentPartType.TEXT, text: message.content }); } const videoFiles = message.extra.filter( (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraVideoFile => extra.type === AttachmentType.VIDEO ); for (const video of videoFiles) { contentParts.push({ type: ContentPartType.INPUT_VIDEO, input_video: { data: video.base64Data, format: video.mimeType.includes('mp4') ? 'mp4' : video.mimeType.includes('ogg') ? 'ogg' : 'auto' } }); } const pdfFiles = message.extra.filter( (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraPdfFile => extra.type === AttachmentType.PDF ); for (const pdfFile of pdfFiles) { if (pdfFile.processedAsImages && pdfFile.images) { for (let i = 0; i < pdfFile.images.length; i++) { contentParts.push({ type: ContentPartType.IMAGE_URL, image_url: { url: pdfFile.images[i] } }); } } else { contentParts.push({ type: ContentPartType.TEXT, text: formatAttachmentText(ATTACHMENT_LABEL_PDF_FILE, pdfFile.name, pdfFile.content) }); } } const mcpPrompts = message.extra.filter( (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpPrompt => extra.type === AttachmentType.MCP_PROMPT ); for (const mcpPrompt of mcpPrompts) { contentParts.push({ type: ContentPartType.TEXT, text: formatAttachmentText( ATTACHMENT_LABEL_MCP_PROMPT, mcpPrompt.name, mcpPrompt.content, mcpPrompt.serverName ) }); } const mcpResources = message.extra.filter( (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraMcpResource => extra.type === AttachmentType.MCP_RESOURCE ); for (const mcpResource of mcpResources) { contentParts.push({ type: ContentPartType.TEXT, text: formatAttachmentText( ATTACHMENT_LABEL_MCP_RESOURCE, mcpResource.name, mcpResource.content, mcpResource.serverName ) }); } const result: ApiChatMessageData = { role: message.role as MessageRole, content: contentParts }; if (message.reasoningContent) { result.reasoning_content = message.reasoningContent; } if (toolCalls && toolCalls.length > 0) { result.tool_calls = toolCalls; } return result; } /** * * * Utilities * * */ /** * Strips legacy inline reasoning content tags from message content. * Handles both plain string content and multipart content arrays. */ private static stripReasoningContent( content: string | ApiChatMessageContentPart[] ): string | ApiChatMessageContentPart[] { const stripFromString = (text: string): string => text.replace(LEGACY_AGENTIC_REGEX.REASONING_BLOCK, '').trim(); if (typeof content === 'string') { return stripFromString(content); } return content.map((part) => { if (part.type === ContentPartType.TEXT && part.text) { return { ...part, text: stripFromString(part.text) }; } return part; }); } /** * Parses error response and creates appropriate error with context information * @param response - HTTP response object * @returns Promise - Parsed error with context info if available */ private static async parseErrorResponse( response: Response ): Promise { try { const errorText = await response.text(); const errorData: ApiErrorResponse = JSON.parse(errorText); const message = errorData.error?.message || 'Unknown server error'; const error = new Error(message) as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number }; }; error.name = response.status === 400 ? 'ServerError' : 'HttpError'; if (errorData.error && 'n_prompt_tokens' in errorData.error && 'n_ctx' in errorData.error) { error.contextInfo = { n_prompt_tokens: errorData.error.n_prompt_tokens, n_ctx: errorData.error.n_ctx }; } return error; } catch { const fallback = new Error( `Server error (${response.status}): ${response.statusText}` ) as Error & { contextInfo?: { n_prompt_tokens: number; n_ctx: number }; }; fallback.name = 'HttpError'; return fallback; } } /** * Extracts model name from Chat Completions API response data. * Handles various response formats including streaming chunks and final responses. * * WORKAROUND: In single model mode, llama-server returns a default/incorrect model name * in the response. We override it with the actual model name from serverStore. * * @param data - Raw response data from the Chat Completions API * @returns Model name string if found, undefined otherwise * @private */ private static extractModelName(data: unknown): string | undefined { const asRecord = (value: unknown): Record | undefined => { return typeof value === 'object' && value !== null ? (value as Record) : undefined; }; const getTrimmedString = (value: unknown): string | undefined => { return typeof value === 'string' && value.trim() ? value.trim() : undefined; }; const root = asRecord(data); if (!root) return undefined; // 1) root (some implementations provide `model` at the top level) const rootModel = getTrimmedString(root.model); if (rootModel) { return rootModel; } // 2) streaming choice (delta) or final response (message) const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; if (!firstChoice) { return undefined; } // priority: delta.model (first chunk) else message.model (final response) const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); if (deltaModel) { return deltaModel; } const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); if (messageModel) { return messageModel; } // avoid guessing from non-standard locations (metadata, etc.) return undefined; } /** * Calls the onTimings callback with timing data from streaming response. * * @param timings - Timing information from the Chat Completions API response * @param promptProgress - Prompt processing progress data * @param onTimingsCallback - Callback function to invoke with timing data * @private */ private static notifyTimings( timings: ChatMessageTimings | undefined, promptProgress: ChatMessagePromptProgress | undefined, onTimingsCallback: | ((timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void) | undefined ): void { if (!onTimingsCallback || (!timings && !promptProgress)) return; onTimingsCallback(timings, promptProgress); } }