import { useEffect, useMemo, useRef, useState } from 'react'; import { BrowserEngineClient, EngineClientError, evaluateModelGate, isShardDownloadFailureDetails, loadModelManifestV2, type EngineCapabilities, type BackendReport, type EngineChatMessage, type EngineChatToolCall, type EngineEvent, type ManifestModelV2, type ModelManifestV2, type ModelTierId, type RequestedBackend, } from '../engine'; import { executeAgentTool, type ToolExecution } from '../lib/agent-tools'; import { normalizeArtifactDocument } from '../lib/artifact-runtime'; import type { ChatMessage, ChatSession, ComposerSubmission, ModelLoadProgress, ModelId, ModelOption, RuntimeSettings, ShardRetryNotice, StorageStatus, StreamState, TelemetrySnapshot, ToolEvent, ToolRun, MessageSegment, } from '../lib/contracts'; import { formatBytes } from '../lib/format'; import { DEFAULT_CONTEXT_SIZE, DEFAULT_MAX_TOKENS } from '../lib/runtime-defaults'; import { conversationTitle, normalizeConversationModelId, planMessageRegeneration, planUserMessageEdit, upsertConversation, } from '../lib/conversations'; import { deleteConversation, listConversations, saveConversation } from '../lib/idb'; import { parseThoughtStream } from '../lib/think-parser'; import { parseToolCalls } from '../lib/tool-protocol'; import { planClientTools } from '../lib/tool-routing'; import { appendToolSegment, normalizeMessageSegments, segmentsForMessage, updateRoundSegments, } from '../lib/message-segments'; import { BonsaiShell } from './components/BonsaiShell'; const MANIFEST_PATH = 'manifest/models.json'; const MAX_TOOL_ROUNDS = 8; const MAX_TOOL_CALLS_PER_ROUND = 4; const MAX_TOOL_CALLS_TOTAL = 20; export const DEFAULT_MODEL_ID: ModelId = 'bonsai-27b'; export { DEFAULT_CONTEXT_SIZE, DEFAULT_MAX_TOKENS } from '../lib/runtime-defaults'; const MODEL_DECOR: Record = { '1_7b': { id: 'bonsai-1.7b', parameters: '1.7B', }, '4b': { id: 'bonsai-4b', parameters: '4B', }, '8b': { id: 'bonsai-8b', parameters: '8B', }, '27b': { id: 'bonsai-27b', parameters: '27B', }, }; export function modelArchitectureLabel(architecture: string): string { if (architecture === 'qwen3') return 'Qwen3 dense'; if (architecture === 'qwen35') return 'Qwen3.5 hybrid'; return architecture; } const UI_TO_TIER: Record = Object.fromEntries( Object.entries(MODEL_DECOR).map(([tier, decor]) => [decor.id, tier]), ) as Record; const INITIAL_SETTINGS: RuntimeSettings = { backend: 'auto', contextSize: DEFAULT_CONTEXT_SIZE, temperature: 0.7, maxTokens: DEFAULT_MAX_TOKENS, systemPrompt: 'You are Bonsai, a private browser-local assistant. Be concise and evidence-led. Use a client tool only when it materially improves the answer, and never claim a tool succeeded before its response arrives.', }; const INITIAL_TELEMETRY: TelemetrySnapshot = { backend: 'Unavailable', device: 'Inspecting browser capabilities…', modelMemoryBytes: 0, kvMemoryBytes: 0, storageUsedBytes: 0, contextUsed: 0, contextLimit: DEFAULT_CONTEXT_SIZE, tokensPerSecond: 0, prefillTokensPerSecond: 0, decodeTokensPerSecond: 0, inferencePhase: 'idle', promptProcessed: 0, promptTotal: 0, completionTokens: 0, timeToFirstTokenMs: 0, gpuLayers: '—', graphSplits: null, }; const INITIAL_STORAGE: StorageStatus = { usedBytes: 0, quotaBytes: 0, persistent: false, downloads: [], }; const INITIAL_MODEL_LOAD_PROGRESS: ModelLoadProgress = { modelId: null, modelLabel: '', stage: 'idle', stageProgress: 0, downloadedBytes: 0, totalBytes: 0, residentBytes: 0, nativeStage: null, shards: [], }; let sharedClient: BrowserEngineClient | null = null; function getEngineClient(): BrowserEngineClient { sharedClient ??= new BrowserEngineClient(); return sharedClient; } function manifestUrl(): string { return new URL(`${import.meta.env.BASE_URL}${MANIFEST_PATH}`, document.baseURI).href; } function clock(): string { return new Date().toLocaleTimeString('en-GB', { hour12: false }); } function eventClock(): string { return new Date().toISOString().slice(11, 23); } function createConversationId(): string { return crypto.randomUUID(); } function errorMessage(error: unknown): string { if (error instanceof EngineClientError) return `${error.code}: ${error.message}`; return error instanceof Error ? error.message : String(error); } function isAborted(error: unknown, signal: AbortSignal): boolean { return signal.aborted || (error instanceof DOMException && error.name === 'AbortError') || (error instanceof EngineClientError && error.code === 'ABORTED'); } export function requiresModelReload(error: unknown): error is EngineClientError { return error instanceof EngineClientError && [ 'WEBGPU_DEVICE_LOST', 'ENGINE_WORKER_FAILED', 'MODEL_NOT_LOADED', ].includes(error.code); } export function reportsModelWeightProgress( phase: Extract['phase'], ): boolean { return phase !== 'manifest'; } export function reconcileCompletionTokens( engineCompletionTokens: number | undefined, streamedTokenEvents: number, ): number { const engineCount = typeof engineCompletionTokens === 'number' && Number.isSafeInteger(engineCompletionTokens) && engineCompletionTokens >= 0 ? engineCompletionTokens : 0; const streamCount = Number.isSafeInteger(streamedTokenEvents) && streamedTokenEvents >= 0 ? streamedTokenEvents : 0; return Math.max(engineCount, streamCount); } export function reconcileTotalTokens( engineTotalTokens: number | undefined, enginePromptTokens: number | undefined, completionTokens: number, ): number { const totalCount = typeof engineTotalTokens === 'number' && Number.isSafeInteger(engineTotalTokens) && engineTotalTokens >= 0 ? engineTotalTokens : 0; const promptCount = typeof enginePromptTokens === 'number' && Number.isSafeInteger(enginePromptTokens) && enginePromptTokens >= 0 ? enginePromptTokens : 0; return Math.max(totalCount, promptCount + completionTokens); } export function publishBackendReport(report: BackendReport): void { globalThis.__bonsaiBackendReport = report; } function appendSegment(current: string, next: string): string { const trimmed = next.trim(); if (!trimmed) return current; return current ? `${current}\n\n${trimmed}` : trimmed; } function replaceMessage( setter: React.Dispatch>, id: string, update: (message: ChatMessage) => ChatMessage, ): void { setter((current) => current.map((message) => message.id === id ? update(message) : message)); } export function applyToolExecutionToMessages( messages: ChatMessage[], assistantId: string, callId: string, execution: ToolExecution, ): ChatMessage[] { const artifactOwnerId = execution.artifact ? messages.find((message) => message.artifacts?.some((artifact) => artifact.id === execution.artifact?.id))?.id : undefined; return messages.map((message) => { const ownsArtifact = message.id === artifactOwnerId; const receivesArtifact = Boolean(execution.artifact) && (ownsArtifact || (!artifactOwnerId && message.id === assistantId)); const artifacts = receivesArtifact && execution.artifact ? ownsArtifact ? (message.artifacts ?? []).map((artifact) => artifact.id === execution.artifact?.id ? execution.artifact : artifact) as NonNullable : [...(message.artifacts ?? []), execution.artifact] : message.artifacts; if (message.id !== assistantId && !receivesArtifact) return message; return { ...message, ...(receivesArtifact ? { artifacts } : {}), ...(message.id === assistantId ? { tools: (message.tools ?? []).map((item) => item.id === callId ? { ...item, output: execution.output, ...(execution.artifact ? { artifactId: execution.artifact.id } : {}), state: execution.failed ? 'error' as const : 'complete' as const, } : item), } : {}), }; }); } export function normalizeStoredMessages(messages: ChatMessage[]): ChatMessage[] { return messages.map((message) => normalizeMessageSegments(message.artifacts ? { ...message, artifacts: message.artifacts.map(normalizeArtifactDocument) } : message)); } export function buildEngineHistory(messages: ChatMessage[], systemPrompt: string): EngineChatMessage[] { const history: EngineChatMessage[] = systemPrompt.trim() ? [{ role: 'system', content: systemPrompt.trim() }] : []; for (const message of messages) { if (message.role === 'user' && message.content.trim() && !message.error) { history.push({ role: 'user', content: message.content }); } else if (message.role === 'assistant' && !message.error) { const completedTools = new Map((message.tools ?? []) .filter((tool) => tool.output !== undefined) .map((tool) => [tool.id, tool])); let content: string[] = []; let tools: ToolRun[] = []; let toolRoundId: string | undefined; const flushAssistantRound = () => { if (content.length === 0 && tools.length === 0) return; history.push({ role: 'assistant', content: content.length > 0 ? content.join('\n\n') : null, ...(tools.length > 0 ? { tool_calls: tools.map((tool) => ({ id: tool.id, type: 'function' as const, function: { name: tool.name, arguments: tool.input }, })), } : {}), }); for (const tool of tools) { history.push({ role: 'tool', content: tool.output ?? '', tool_call_id: tool.id }); } content = []; tools = []; toolRoundId = undefined; }; for (const segment of segmentsForMessage(message)) { if (segment.kind === 'reasoning') { if (tools.length > 0) flushAssistantRound(); continue; } if (segment.kind === 'text') { if (tools.length > 0) flushAssistantRound(); if (segment.content.trim()) content.push(segment.content.trim()); continue; } if (segment.kind !== 'tool') continue; const tool = completedTools.get(segment.toolCallId); if (!tool) continue; if (tools.length > 0 && segment.roundId && toolRoundId && segment.roundId !== toolRoundId) { flushAssistantRound(); } toolRoundId = segment.roundId; tools.push(tool); } flushAssistantRound(); } } return history; } function fallbackToolCalls(text: string): EngineChatToolCall[] { return parseToolCalls(text).calls.map((call) => ({ id: crypto.randomUUID(), type: 'function', function: { name: call.name, arguments: JSON.stringify(call.arguments), }, })); } export function App() { const [client] = useState(getEngineClient); const [manifest, setManifest] = useState(null); const [capabilities, setCapabilities] = useState(null); const [activeModelId, setActiveModelId] = useState(DEFAULT_MODEL_ID); const [loadedModelId, setLoadedModelId] = useState(null); const [loadingModelId, setLoadingModelId] = useState(null); const [messages, setMessages] = useState([]); const [conversations, setConversations] = useState([]); const [activeConversationId, setActiveConversationId] = useState(createConversationId); const [conversationListOpen, setConversationListOpen] = useState(false); const [streamState, setStreamState] = useState('idle'); const [modelLoadProgress, setModelLoadProgress] = useState(INITIAL_MODEL_LOAD_PROGRESS); const [toolsEnabled, setToolsEnabled] = useState(false); const [settings, setSettings] = useState(INITIAL_SETTINGS); const [settingsOpen, setSettingsOpen] = useState(false); const [storage, setStorage] = useState(INITIAL_STORAGE); const [telemetry, setTelemetry] = useState(INITIAL_TELEMETRY); const [toolEvents, setToolEvents] = useState([]); const [error, setError] = useState(null); const [shardRetry, setShardRetry] = useState(null); const [bootStatus, setBootStatus] = useState('Inspecting local runtime…'); const operationRef = useRef(null); const loadedConfigurationRef = useRef<{ id: ModelId; backend: RequestedBackend; contextSize: number } | null>(null); const historyReadyRef = useRef(false); const loadProgressFrameRef = useRef(null); const pendingLoadProgressRef = useRef(null); const modelLoadProgressRef = useRef(INITIAL_MODEL_LOAD_PROGRESS); const inferenceFrameRef = useRef(null); const pendingInferenceRef = useRef | null>(null); const publishModelLoadProgress = (next: ModelLoadProgress): void => { modelLoadProgressRef.current = next; pendingLoadProgressRef.current = next; if (loadProgressFrameRef.current !== null) return; loadProgressFrameRef.current = window.requestAnimationFrame(() => { loadProgressFrameRef.current = null; const pending = pendingLoadProgressRef.current; pendingLoadProgressRef.current = null; if (pending) setModelLoadProgress(pending); }); }; const publishInferenceProgress = ( next: Extract, ): void => { pendingInferenceRef.current = next; if (inferenceFrameRef.current !== null) return; inferenceFrameRef.current = window.requestAnimationFrame(() => { inferenceFrameRef.current = null; const pending = pendingInferenceRef.current; pendingInferenceRef.current = null; if (!pending) return; const liveRate = pending.phase === 'prefill' ? pending.promptTokensPerSecond : pending.decodeTokensPerSecond; setTelemetry((current) => ({ ...current, inferencePhase: pending.phase, tokensPerSecond: liveRate, prefillTokensPerSecond: pending.promptTokensPerSecond || current.prefillTokensPerSecond, decodeTokensPerSecond: pending.decodeTokensPerSecond || current.decodeTokensPerSecond, promptProcessed: pending.promptProcessed, promptTotal: pending.promptTotal, completionTokens: pending.completionTokens, contextUsed: Math.min( current.contextLimit, Math.max(current.contextUsed, pending.promptProcessed + pending.completionTokens), ), })); }); }; const clearPendingInferenceProgress = (): void => { if (inferenceFrameRef.current !== null) { window.cancelAnimationFrame(inferenceFrameRef.current); inferenceFrameRef.current = null; } pendingInferenceRef.current = null; }; const handleModelInvalidation = (runtimeError: unknown): boolean => { if (!requiresModelReload(runtimeError)) return false; setLoadedModelId(null); loadedConfigurationRef.current = null; setTelemetry((current) => ({ ...current, backend: 'Unavailable', modelMemoryBytes: 0, kvMemoryBytes: 0, contextUsed: 0, tokensPerSecond: 0, prefillTokensPerSecond: 0, decodeTokensPerSecond: 0, inferencePhase: 'idle', promptProcessed: 0, promptTotal: 0, completionTokens: 0, gpuLayers: '—', graphSplits: null, })); setError(errorMessage(runtimeError)); setBootStatus(runtimeError.code === 'WEBGPU_DEVICE_LOST' ? 'WebGPU device lost · model unloaded · run again to reload' : 'Browser engine restarted · model unloaded · run again to reload'); return true; }; const persistenceAttemptedRef = useRef(false); useEffect(() => () => { if (loadProgressFrameRef.current !== null) { window.cancelAnimationFrame(loadProgressFrameRef.current); } if (inferenceFrameRef.current !== null) { window.cancelAnimationFrame(inferenceFrameRef.current); } }, []); useEffect(() => { let cancelled = false; const bootstrap = async () => { let loadedManifest: ModelManifestV2 | null = null; try { loadedManifest = await loadModelManifestV2(manifestUrl()); } catch (bootstrapError) { if (!cancelled) setError(`Manifest unavailable: ${errorMessage(bootstrapError)}`); } try { const nextCapabilities = await client.capabilities(); if (!cancelled) { setCapabilities(nextCapabilities); setStorage({ usedBytes: nextCapabilities.storage.usageBytes ?? 0, quotaBytes: nextCapabilities.storage.quotaBytes ?? 0, persistent: nextCapabilities.storage.persisted, downloads: [], }); setTelemetry((current) => ({ ...current, device: nextCapabilities.webgpu.name || (nextCapabilities.webgpu.available ? 'WebGPU adapter' : 'CPU · WebAssembly'), storageUsedBytes: nextCapabilities.storage.usageBytes ?? 0, })); setBootStatus(nextCapabilities.webgpu.available ? `${nextCapabilities.webgpu.name || 'WebGPU'} ready · model runs after local load` : 'WebGPU unavailable · dense tiers can use CPU-WASM'); } } catch (capabilityError) { if (!cancelled) setError(`Capability inspection failed: ${errorMessage(capabilityError)}`); } try { const storedConversations = (await listConversations()).map((conversation) => ({ ...conversation, messages: normalizeStoredMessages(conversation.messages), modelId: normalizeConversationModelId(conversation.modelId), systemPrompt: typeof conversation.systemPrompt === 'string' ? conversation.systemPrompt : INITIAL_SETTINGS.systemPrompt, })); const latest = storedConversations[0]; if (!cancelled) { setConversations(storedConversations); if (latest) { setActiveConversationId(latest.id); setActiveModelId(latest.modelId); setMessages(latest.messages); setSettings((current) => ({ ...current, systemPrompt: latest.systemPrompt })); } } } catch { // Private browsing may disable IndexedDB. Chat remains usable for the tab. if (!cancelled) setBootStatus((current) => `${current} · history is session-only`); } finally { historyReadyRef.current = true; if (!cancelled && loadedManifest) setManifest(loadedManifest); } }; void bootstrap(); return () => { cancelled = true; }; }, [client]); useEffect(() => { if (!historyReadyRef.current) return; const alreadySaved = conversations.some((conversation) => conversation.id === activeConversationId); if (messages.length === 0 && settings.systemPrompt === INITIAL_SETTINGS.systemPrompt && !alreadySaved) return; const nextConversation: ChatSession = { id: activeConversationId, title: conversationTitle(messages), messages, modelId: activeModelId, systemPrompt: settings.systemPrompt, updatedAt: Date.now(), }; const timer = window.setTimeout(() => { setConversations((current) => upsertConversation(current, nextConversation)); void saveConversation(nextConversation).catch(() => setBootStatus((current) => current.includes('history is session-only') ? current : `${current} · history is session-only`)); }, 250); return () => window.clearTimeout(timer); }, [activeConversationId, activeModelId, messages, settings.systemPrompt]); const models = useMemo(() => { if (!manifest) return []; return manifest.models.map((model) => { const decor = MODEL_DECOR[model.id]; const configured = loadedConfigurationRef.current; const cachedDownload = storage.downloads.find((download) => ( download.modelId === decor.id && download.state === 'complete' )); const configuredContext = model.id === '27b' ? Math.min(settings.contextSize, model.defaultContext) : settings.contextSize; const isLoaded = loadedModelId === decor.id && configured?.id === decor.id && configured.backend === settings.backend && configured.contextSize === configuredContext; let availability: ModelOption['availability'] = isLoaded ? 'loaded' : loadingModelId === decor.id ? 'loading' : cachedDownload ? 'cached' : 'remote'; let limitReason: string | undefined; if (!isLoaded && capabilities) { const gate = evaluateModelGate( model, settings.backend, capabilities.webgpu, capabilities.storage, cachedDownload?.totalBytes ?? 0, ); if (!gate.allowed) { availability = 'limited'; limitReason = gate.reasons.join(' '); } } return { id: decor.id, manifestId: model.id, label: model.displayName, architectureLabel: modelArchitectureLabel(model.architecture), parameters: decor.parameters, runtimeLabel: model.cpuFallback ? 'WebGPU · WASM fallback' : 'WebGPU required', footprint: formatBytes(model.downloadBytes), availability, contextLimit: model.contextLength, defaultContext: model.defaultContext, ...(limitReason ? { limitReason } : {}), }; }); }, [capabilities, loadedModelId, loadingModelId, manifest, settings.backend, settings.contextSize, storage.downloads]); const activeTier = UI_TO_TIER[activeModelId]; const refreshStorage = async (): Promise => { const estimate = await client.storageEstimate(); setStorage((current) => ({ ...current, usedBytes: estimate.usageBytes ?? 0, quotaBytes: estimate.quotaBytes ?? 0, persistent: estimate.persisted, })); setCapabilities((current) => current ? { ...current, storage: estimate } : current); setTelemetry((current) => ({ ...current, storageUsedBytes: estimate.usageBytes ?? 0 })); }; const requestPersistentStorage = async (force = false): Promise => { if (storage.persistent || (!force && persistenceAttemptedRef.current)) return; persistenceAttemptedRef.current = true; setBootStatus('Requesting persistent browser storage before download…'); try { const estimate = await client.storagePersist(); setStorage((current) => ({ ...current, usedBytes: estimate.usageBytes ?? 0, quotaBytes: estimate.quotaBytes ?? 0, persistent: estimate.persisted, })); setCapabilities((current) => current ? { ...current, storage: estimate } : current); setTelemetry((current) => ({ ...current, storageUsedBytes: estimate.usageBytes ?? 0 })); setBootStatus(estimate.granted ? 'Persistent browser storage granted' : 'Persistent storage was not granted · continuing with browser-managed storage'); } catch (persistError) { if (handleModelInvalidation(persistError)) return; setBootStatus('Persistent storage request unavailable · continuing with browser-managed storage'); } }; const loadModel = async ( modelId: ModelId, controller: AbortController, contextSize: number, ): Promise => { if (!manifest) throw new Error('The pinned model manifest is still loading.'); const tier = UI_TO_TIER[modelId]; const model = manifest.models.find((candidate) => candidate.id === tier); if (!model) throw new Error(`Model ${tier} is absent from the pinned manifest.`); setLoadedModelId(null); loadedConfigurationRef.current = null; setLoadingModelId(modelId); setStreamState('loading'); setError(null); setShardRetry(null); setBootStatus(`Preparing ${model.displayName}…`); const initialProgress: ModelLoadProgress = { ...INITIAL_MODEL_LOAD_PROGRESS, modelId, modelLabel: model.displayName, stage: 'manifest', totalBytes: model.downloadBytes, }; modelLoadProgressRef.current = initialProgress; setModelLoadProgress(initialProgress); try { const result = await client.loadModel({ manifestUrl: manifestUrl(), modelId: tier, backend: settings.backend, contextSize, }, { signal: controller.signal, onProgress: (progress) => { const received = Math.min(progress.loadedBytes, model.downloadBytes); const shard = progress.shardIndex === null ? '' : ` · shard ${progress.shardIndex + 1}/${progress.shardCount}`; const stageProgress = progress.stageProgress ?? (progress.totalBytes > 0 ? progress.loadedBytes / progress.totalBytes : 0); publishModelLoadProgress({ modelId, modelLabel: model.displayName, stage: progress.phase, stageProgress: Math.min(1, Math.max(0, stageProgress)), downloadedBytes: received, totalBytes: model.downloadBytes, residentBytes: progress.residentBytes ?? 0, nativeStage: progress.nativeStage, shards: progress.shards.map((item) => ({ ...item })), }); if (progress.residentBytes !== null) { setTelemetry((current) => ({ ...current, modelMemoryBytes: progress.residentBytes ?? 0 })); } if (!reportsModelWeightProgress(progress.phase)) { setBootStatus('Reading pinned model manifest…'); return; } const stageLabel = progress.phase === 'cache' ? 'Inspecting cache' : progress.phase === 'verify' ? 'Verifying shards' : progress.phase === 'load' ? 'Allocating model' : 'Downloading model'; setBootStatus(`${stageLabel}${shard} · ${Math.round(stageProgress * 100)}%`); if (progress.phase !== 'load') { setStorage((current) => ({ ...current, downloads: [ ...current.downloads.filter((download) => download.modelId !== modelId), { modelId, label: model.displayName, receivedBytes: received, totalBytes: model.downloadBytes, state: received >= model.downloadBytes ? 'complete' : 'downloading', }, ], })); } }, }); setLoadedModelId(modelId); setShardRetry(null); loadedConfigurationRef.current = { id: modelId, backend: settings.backend, contextSize }; setTelemetry((current) => ({ ...current, backend: result.backend === 'webgpu' ? 'WebGPU' : 'WASM', device: result.backend === 'webgpu' ? capabilities?.webgpu.name || 'WebGPU adapter' : `CPU · ${capabilities?.hardwareConcurrency ?? navigator.hardwareConcurrency ?? 1} logical cores`, modelMemoryBytes: result.backendReport.modelBufferBytes ?? model.downloadBytes, kvMemoryBytes: result.backendReport.webgpuKvBufferBytes ?? 0, contextUsed: 0, contextLimit: result.context.size, gpuLayers: result.backendReport.layersGpu ? `${result.backendReport.layersGpu.offloaded} / ${result.backendReport.layersGpu.total}` : result.backend === 'wasm' ? 'CPU' : 'unreported', graphSplits: result.backendReport.nGraphSplits, })); publishModelLoadProgress({ ...modelLoadProgressRef.current, modelId, modelLabel: model.displayName, stage: 'ready', stageProgress: 1, downloadedBytes: model.downloadBytes, totalBytes: model.downloadBytes, residentBytes: result.backendReport.allocatedBufferBytes ?? result.backendReport.modelBufferBytes ?? model.downloadBytes, nativeStage: null, }); setBootStatus(`${model.displayName} loaded · ${result.backend.toUpperCase()} · ${result.context.size.toLocaleString()} ctx`); await refreshStorage(); } catch (loadError) { const operatorAborted = controller.signal.aborted; const retryDetails = !operatorAborted && loadError instanceof EngineClientError && isShardDownloadFailureDetails(loadError.details) ? loadError.details : null; if (operatorAborted) setShardRetry(null); if (retryDetails) { setShardRetry({ modelId, shardIndex: retryDetails.shardIndex, shardCount: retryDetails.shardCount, shardPath: retryDetails.shardPath, error: errorMessage(loadError), }); } publishModelLoadProgress({ ...modelLoadProgressRef.current, stage: operatorAborted ? 'idle' : 'error', nativeStage: null, }); setStorage((current) => ({ ...current, downloads: operatorAborted ? current.downloads.filter((download) => download.modelId !== modelId) : current.downloads.map((download) => download.modelId === modelId ? { ...download, label: retryDetails ? `${model.displayName} · shard ${retryDetails.shardIndex + 1}/${retryDetails.shardCount} · ${retryDetails.shardPath}` : download.label, state: 'error', } : download), })); throw loadError; } finally { setLoadingModelId(null); } }; const targetContext = (model: ManifestModelV2): number => { if (model.id === activeTier) { const provenLimit = model.id === '27b' ? model.defaultContext : model.contextLength; return Math.min(settings.contextSize, provenLimit); } return Math.min(DEFAULT_CONTEXT_SIZE, model.defaultContext); }; const handleModelSelect = (modelId: ModelId) => { if (operationRef.current) return; setShardRetry(null); setActiveModelId(modelId); const model = manifest?.models.find((candidate) => candidate.id === UI_TO_TIER[modelId]); if (model) { setSettings((current) => ({ ...current, backend: model.cpuFallback ? current.backend : current.backend === 'wasm' ? 'auto' : current.backend, contextSize: Math.min(DEFAULT_CONTEXT_SIZE, model.defaultContext), })); } }; const handleLoadModel = (modelId: ModelId) => { if (operationRef.current || !manifest) return; setActiveModelId(modelId); const model = manifest.models.find((candidate) => candidate.id === UI_TO_TIER[modelId]); if (!model) return; const contextSize = targetContext(model); if (model.id !== activeTier) setSettings((current) => ({ ...current, contextSize })); const controller = new AbortController(); operationRef.current = controller; void requestPersistentStorage() .then(() => { if (controller.signal.aborted) throw new DOMException('Model loading was aborted.', 'AbortError'); return loadModel(modelId, controller, contextSize); }) .then(() => setStreamState('complete')) .catch((loadError) => { if (handleModelInvalidation(loadError)) { setStreamState('error'); return; } if (!isAborted(loadError, controller.signal)) { const steer = model.id === '27b' ? ' Choose Bonsai 8B for the supported fallback path.' : ''; setError(`${errorMessage(loadError)}${steer}`); setStreamState('error'); setBootStatus('Model load failed safely'); } else { setStreamState('idle'); setBootStatus('Load stopped by operator'); } }) .finally(() => { if (operationRef.current === controller) operationRef.current = null; }); }; const startSubmission = ( { prompt, toolsEnabled: submissionToolsEnabled }: ComposerSubmission, baseMessages: ChatMessage[], existingUserMessage?: ChatMessage, ): boolean => { if (operationRef.current || !manifest) return false; const tier = UI_TO_TIER[activeModelId]; const selectedModel = manifest.models.find((candidate) => candidate.id === tier); if (!selectedModel) { setError(`Model ${tier} is absent from the pinned manifest.`); return false; } const requestedContext = targetContext(selectedModel); const configured = loadedConfigurationRef.current; const needsLoad = loadedModelId !== activeModelId || configured?.backend !== settings.backend || configured.contextSize !== requestedContext; if (needsLoad) { setError(`Load ${selectedModel.displayName} before running this prompt.`); setBootStatus('Explicit model load required'); return false; } const controller = new AbortController(); operationRef.current = controller; const nonce = crypto.randomUUID(); const userMessage: ChatMessage = existingUserMessage ?? { id: `user-${nonce}`, role: 'user', content: prompt, timestamp: clock(), }; const assistantId = `assistant-${nonce}`; const assistantMessage: ChatMessage = { id: assistantId, role: 'assistant', content: '', reasoning: '', timestamp: clock(), state: 'streaming', tools: [], segments: [], }; const historySnapshot = [...baseMessages, userMessage]; setMessages([...historySnapshot, assistantMessage]); setError(null); const run = async () => { setStreamState('streaming'); setBootStatus(`Generating with ${selectedModel.displayName}…`); const initialToolPlan = planClientTools(prompt, submissionToolsEnabled, 0); const routedSystemPrompt = [settings.systemPrompt, initialToolPlan.systemInstruction] .filter((value): value is string => Boolean(value?.trim())) .join('\n\n'); const engineMessages = buildEngineHistory(historySnapshot, routedSystemPrompt); let visibleText = ''; let reasoningText = ''; let totalCompletionTokens = 0; let totalPromptTokens = 0; let lastTotalTokens = 0; let tokensPerSecond = 0; let prefillTokensPerSecond = 0; let totalPromptSeconds = 0; let totalDecodeSeconds = 0; let firstTokenAt = 0; let outputError: string | null = null; const generationStartedAt = performance.now(); let toolRounds = 0; let totalToolCalls = 0; let roundSequence = 0; let messageSegments: MessageSegment[] = []; const artifactRegistry = new Map(historySnapshot.flatMap((message) => ( message.artifacts ?? [] )).map((artifact) => [artifact.id, artifact])); clearPendingInferenceProgress(); setTelemetry((current) => ({ ...current, inferencePhase: 'prefill', tokensPerSecond: 0, prefillTokensPerSecond: 0, decodeTokensPerSecond: 0, promptProcessed: 0, promptTotal: 0, completionTokens: 0, timeToFirstTokenMs: 0, })); while (true) { const roundId = `round-${roundSequence}`; roundSequence += 1; const toolPlan = planClientTools(prompt, submissionToolsEnabled, toolRounds); let streamedText = ''; let streamedReasoning = ''; let streamedTokenEvents = 0; const completionOffset = totalCompletionTokens; const result = await client.generate({ messages: engineMessages, maxTokens: Math.max(settings.maxTokens, toolPlan.minimumMaxTokens), temperature: settings.temperature, topK: settings.temperature === 0 ? 1 : 40, topP: settings.temperature === 0 ? 1 : 0.9, ...(toolPlan.tools ? { tools: toolPlan.tools } : {}), toolChoice: toolPlan.toolChoice, cachePrompt: true, }, { signal: controller.signal, onGenerationProgress: (progress) => { publishInferenceProgress({ ...progress, completionTokens: completionOffset + progress.completionTokens, }); }, onToken: (delta, reasoningDelta) => { if (!firstTokenAt && (delta || reasoningDelta)) firstTokenAt = performance.now(); if (delta || reasoningDelta) streamedTokenEvents += 1; streamedText += delta; streamedReasoning += reasoningDelta ?? ''; const thought = parseThoughtStream(streamedText); const toolText = parseToolCalls(thought.content); const currentReasoning = streamedReasoning || thought.reasoning; messageSegments = updateRoundSegments( messageSegments, roundId, currentReasoning, toolText.content, ); replaceMessage(setMessages, assistantId, (message) => ({ ...message, content: appendSegment(visibleText, toolText.content), reasoning: appendSegment(reasoningText, currentReasoning), segments: messageSegments, isThinking: thought.isThinking || Boolean(reasoningDelta), state: 'streaming', })); }, onToolCallProgress: (activity) => { if (!firstTokenAt) firstTokenAt = performance.now(); replaceMessage(setMessages, assistantId, (message) => ({ ...message, toolActivity: { name: activity.name || message.toolActivity?.name || 'tool call', argumentCharacters: activity.argumentCharacters, }, state: 'streaming', })); }, }); const roundCompletionTokens = reconcileCompletionTokens( result.usage?.completionTokens, streamedTokenEvents, ); const roundPromptTokens = result.usage?.promptTokens ?? 0; totalPromptTokens += roundPromptTokens; totalCompletionTokens += roundCompletionTokens; lastTotalTokens = Math.max( lastTotalTokens, reconcileTotalTokens( result.usage?.totalTokens, result.usage?.promptTokens, roundCompletionTokens, ), ); const roundPrefillRate = result.timings?.promptTokensPerSecond ?? 0; const roundDecodeRate = result.timings?.predictedTokensPerSecond ?? 0; if (roundPromptTokens > 0 && roundPrefillRate > 0) { totalPromptSeconds += roundPromptTokens / roundPrefillRate; } if (roundCompletionTokens > 0 && roundDecodeRate > 0) { totalDecodeSeconds += roundCompletionTokens / roundDecodeRate; } const thought = parseThoughtStream(result.text); const textual = parseToolCalls(thought.content); if (textual.incomplete || textual.errors.length > 0) { throw new Error(`Malformed textual tool call: ${textual.errors.join('; ') || 'closing is missing'}`); } visibleText = appendSegment(visibleText, textual.content); reasoningText = appendSegment(reasoningText, streamedReasoning || thought.reasoning); messageSegments = updateRoundSegments( messageSegments, roundId, streamedReasoning || thought.reasoning, textual.content, ); const calls = result.toolCalls.length > 0 ? result.toolCalls : fallbackToolCalls(thought.content); if (toolPlan.requiredToolName && calls.length === 0) { outputError = (result.toolCallError || result.finishReason === 'length') ? `The ${toolPlan.requiredToolName} call ended before its arguments were complete. Regenerate, shorten the request, or clear earlier turns so the ${DEFAULT_MAX_TOKENS.toLocaleString()}-token completion ceiling fits the context.` : `The model did not call the required ${toolPlan.requiredToolName} tool for this artifact request.`; } if (toolPlan.requiredToolName && calls.some((call) => call.function.name !== toolPlan.requiredToolName)) { throw new Error(`The model returned a different tool when ${toolPlan.requiredToolName} was required.`); } replaceMessage(setMessages, assistantId, (message) => ({ ...message, content: visibleText, reasoning: reasoningText, segments: messageSegments, isThinking: false, toolActivity: undefined, })); if (outputError) break; engineMessages.push({ role: 'assistant', content: textual.content || null, ...(calls.length > 0 ? { tool_calls: calls } : {}), }); if (calls.length === 0) break; if (!submissionToolsEnabled) throw new Error('The model requested a tool while tools were disabled; nothing was executed.'); if (toolRounds >= MAX_TOOL_ROUNDS) throw new Error(`Tool loop stopped at the ${MAX_TOOL_ROUNDS}-round safety limit.`); if (calls.length > MAX_TOOL_CALLS_PER_ROUND) { throw new Error(`Tool round requested ${calls.length} calls; the limit is ${MAX_TOOL_CALLS_PER_ROUND}.`); } totalToolCalls += calls.length; if (totalToolCalls > MAX_TOOL_CALLS_TOTAL) { throw new Error(`Tool loop requested more than the ${MAX_TOOL_CALLS_TOTAL}-call session limit.`); } toolRounds += 1; for (const call of calls) { messageSegments = appendToolSegment(messageSegments, call.id, roundId); const toolRun: ToolRun = { id: call.id, name: call.function.name, input: call.function.arguments, state: 'running', }; const event: ToolEvent = { id: call.id, timestamp: eventClock(), label: call.function.name, detail: `round ${toolRounds} · executing locally`, state: 'running', }; replaceMessage(setMessages, assistantId, (message) => ({ ...message, tools: [...(message.tools ?? []), toolRun], segments: messageSegments, })); setToolEvents((current) => [...current.filter((item) => item.id !== call.id), event].slice(-20)); const execution = await executeAgentTool(call, controller.signal, { artifacts: artifactRegistry }); if (controller.signal.aborted) throw new DOMException('The tool operation was aborted.', 'AbortError'); setMessages((current) => applyToolExecutionToMessages(current, assistantId, call.id, execution)); setToolEvents((current) => current.map((item) => item.id === call.id ? { ...item, timestamp: eventClock(), detail: execution.failed ? 'tool returned a bounded error' : 'local result returned to model', state: execution.failed ? 'error' : 'complete', } : item)); engineMessages.push({ role: 'tool', content: execution.output, tool_call_id: call.id }); } } const generationElapsedSeconds = Math.max(0.001, (performance.now() - generationStartedAt) / 1000); tokensPerSecond = totalDecodeSeconds > 0 ? totalCompletionTokens / totalDecodeSeconds : totalCompletionTokens / generationElapsedSeconds; prefillTokensPerSecond = totalPromptSeconds > 0 ? totalPromptTokens / totalPromptSeconds : 0; const ttft = firstTokenAt ? Math.round(firstTokenAt - generationStartedAt) : 0; const report = await client.backendReport(); publishBackendReport(report); clearPendingInferenceProgress(); replaceMessage(setMessages, assistantId, (message) => ({ ...message, state: outputError ? 'error' : 'complete', isThinking: false, toolActivity: undefined, ...(outputError ? { error: outputError } : {}), metrics: { tokens: totalCompletionTokens, tokensPerSecond, promptTokensPerSecond: prefillTokensPerSecond, timeToFirstTokenMs: ttft, }, })); setTelemetry((current) => ({ ...current, contextUsed: Math.min(current.contextLimit, Math.max(lastTotalTokens, totalCompletionTokens)), tokensPerSecond, prefillTokensPerSecond, decodeTokensPerSecond: tokensPerSecond, inferencePhase: 'complete', promptProcessed: totalPromptTokens, promptTotal: totalPromptTokens, completionTokens: totalCompletionTokens, timeToFirstTokenMs: ttft, graphSplits: report.nGraphSplits, gpuLayers: report.layersGpu ? `${report.layersGpu.offloaded} / ${report.layersGpu.total}` : current.gpuLayers, })); setStreamState(outputError ? 'error' : 'complete'); setBootStatus(outputError ? `${selectedModel.displayName} stopped at an incomplete tool call` : `${selectedModel.displayName} complete · ${tokensPerSecond.toFixed(1)} tok/s`); await refreshStorage(); }; void run() .catch((runError) => { clearPendingInferenceProgress(); if (handleModelInvalidation(runError)) { const message = errorMessage(runError); replaceMessage(setMessages, assistantId, (current) => ({ ...current, state: 'error', isThinking: false, toolActivity: undefined, error: message, })); setStreamState('error'); return; } if (isAborted(runError, controller.signal)) { replaceMessage(setMessages, assistantId, (message) => ({ ...message, state: 'complete', isThinking: false, toolActivity: undefined, })); setStreamState('complete'); setTelemetry((current) => ({ ...current, inferencePhase: 'idle' })); setBootStatus('Generation stopped by operator'); return; } const message = errorMessage(runError); replaceMessage(setMessages, assistantId, (current) => ({ ...current, state: 'error', isThinking: false, toolActivity: undefined, error: message, })); setError(message); setStreamState('error'); setTelemetry((current) => ({ ...current, inferencePhase: 'idle' })); setBootStatus('Runtime stopped safely'); }) .finally(() => { if (operationRef.current === controller) operationRef.current = null; }); return true; }; const handleSend = (submission: ComposerSubmission) => { startSubmission(submission, messages); }; const handleRegenerate = (messageId: string) => { if (operationRef.current) return; const plan = planMessageRegeneration(messages, messageId); if (!plan) return; const started = startSubmission( { prompt: plan.userMessage.content, toolsEnabled }, plan.baseMessages, plan.userMessage, ); if (!started) return; const retainedToolIds = new Set(plan.baseMessages.flatMap((message) => ( message.tools?.map((tool) => tool.id) ?? [] ))); setToolEvents((current) => current.filter((event) => retainedToolIds.has(event.id))); }; const handleEditUserMessage = (messageId: string, content: string) => { if (operationRef.current) return; const plan = planUserMessageEdit(messages, messageId, content); if (!plan) return; const started = startSubmission( { prompt: plan.userMessage.content, toolsEnabled }, plan.baseMessages, plan.userMessage, ); if (!started) return; const retainedToolIds = new Set(plan.baseMessages.flatMap((message) => ( message.tools?.map((tool) => tool.id) ?? [] ))); setToolEvents((current) => current.filter((event) => retainedToolIds.has(event.id))); }; const handleCancel = () => { operationRef.current?.abort(); }; const handlePersistStorage = () => { void requestPersistentStorage(true); }; const handleClearStorage = () => { if (operationRef.current) return; if (!window.confirm('Delete every cached Bonsai model from this browser profile? Conversation and memory notes are kept.')) return; void client.storageClear() .then((estimate) => { setLoadedModelId(null); loadedConfigurationRef.current = null; setStorage({ usedBytes: estimate.usageBytes ?? 0, quotaBytes: estimate.quotaBytes ?? 0, persistent: estimate.persisted, downloads: [], }); setTelemetry((current) => ({ ...INITIAL_TELEMETRY, device: current.device })); setBootStatus('Cached model files deleted'); }) .catch((clearError) => { if (!handleModelInvalidation(clearError)) setError(errorMessage(clearError)); }); }; const handleClearConversation = () => { if (operationRef.current) return; const clearedId = activeConversationId; setConversations((current) => current.filter((conversation) => conversation.id !== clearedId)); setActiveConversationId(createConversationId()); setMessages([]); setToolEvents([]); setBootStatus('Conversation cleared · model remains local'); void deleteConversation(clearedId).catch(() => setBootStatus('Conversation cleared for this tab · local history is unavailable')); }; const handleOpenConversationList = () => { if (operationRef.current) return; const alreadySaved = conversations.some((conversation) => conversation.id === activeConversationId); if (messages.length > 0 || settings.systemPrompt !== INITIAL_SETTINGS.systemPrompt || alreadySaved) { const nextConversation: ChatSession = { id: activeConversationId, title: conversationTitle(messages), messages, modelId: activeModelId, systemPrompt: settings.systemPrompt, updatedAt: Date.now(), }; setConversations((current) => upsertConversation(current, nextConversation)); void saveConversation(nextConversation).catch(() => undefined); } setConversationListOpen(true); }; const handleNewConversation = () => { if (operationRef.current) return; setActiveConversationId(createConversationId()); setMessages([]); setToolEvents([]); setError(null); setStreamState('idle'); setSettings((current) => ({ ...current, systemPrompt: INITIAL_SETTINGS.systemPrompt })); setConversationListOpen(false); setBootStatus('New local chat · model remains available'); }; const handleOpenConversation = (conversationId: string) => { if (operationRef.current) return; const conversation = conversations.find((candidate) => candidate.id === conversationId); if (!conversation) return; setActiveConversationId(conversation.id); setMessages(conversation.messages); setToolEvents([]); setError(null); setStreamState('idle'); setSettings((current) => ({ ...current, systemPrompt: conversation.systemPrompt })); handleModelSelect(conversation.modelId); setConversationListOpen(false); setBootStatus(`Opened local chat · ${conversation.title}`); }; const handleOpenConversationSettings = (conversationId: string) => { if (operationRef.current) return; handleOpenConversation(conversationId); setSettingsOpen(true); }; const handleDeleteConversation = (conversationId: string) => { if (operationRef.current) return; const conversation = conversations.find((candidate) => candidate.id === conversationId); if (!conversation || !window.confirm(`Delete “${conversation.title}” from this browser?`)) return; setConversations((current) => current.filter((candidate) => candidate.id !== conversationId)); if (conversationId === activeConversationId) { setActiveConversationId(createConversationId()); setMessages([]); setToolEvents([]); setSettings((current) => ({ ...current, systemPrompt: INITIAL_SETTINGS.systemPrompt })); } void deleteConversation(conversationId) .then(() => setBootStatus('Local chat deleted')) .catch(() => setBootStatus('Chat removed for this tab · local history is unavailable')); }; const effectiveLoadedModelId = (() => { const configured = loadedConfigurationRef.current; const loadedModel = manifest?.models.find((model) => model.id === UI_TO_TIER[loadedModelId ?? activeModelId]); const configuredContext = loadedModel?.id === '27b' ? Math.min(settings.contextSize, loadedModel.defaultContext) : settings.contextSize; return configured && configured.id === loadedModelId && configured.backend === settings.backend && configured.contextSize === configuredContext ? loadedModelId : null; })(); if (models.length === 0) { return (
); } return ( { if (!operationRef.current) setSettingsOpen(true); }} onCloseSettings={() => setSettingsOpen(false)} onSettingsChange={(nextSettings) => { if (!operationRef.current) setSettings(nextSettings); }} onPersistStorage={handlePersistStorage} onClearStorage={handleClearStorage} onClearConversation={handleClearConversation} onOpenConversationList={handleOpenConversationList} onNewConversation={handleNewConversation} onOpenConversation={handleOpenConversation} onOpenConversationSettings={handleOpenConversationSettings} onDeleteConversation={handleDeleteConversation} onDismissError={() => setError(null)} onRetryShard={handleLoadModel} /> ); }