import { useEffect, useMemo, useRef, useState } from 'react'; import { BrowserEngineClient, EngineClientError, evaluateModelContextPolicy, evaluateModelGate, loadModelManifestV2, type BenchmarkFlashMode, type BenchmarkKvCacheType, type BenchmarkWasmFlavor, type EngineCapabilities, type GenerateParams, type ModelManifestV2, type ModelTierId, type RequestedBackend, } from '../engine'; import { formatBytes } from '../lib/format'; import { BENCHMARK_WORKLOADS, DEFAULT_BENCHMARK_WORKLOAD_ID, benchmarkReportFilename, buildBenchmarkReport, serializeBenchmarkReport, type BenchmarkReport, type BenchmarkWorkloadId, type TeacherForcedReferenceScoreEvidence, } from './report'; import { STATE_DRIFT_REFERENCE_PATH, validateStateDriftReferenceFixture, } from './state-drift-reference'; const MANIFEST_PATH = 'manifest/models.json'; const DEFAULT_MODEL: ModelTierId = '1_7b'; const MAX_GLUE_INT = 2_147_483_647; interface RunProgress { label: string; loadedBytes: number; totalBytes: number; } let sharedBenchClient: BrowserEngineClient | null = null; function getBenchClient(): BrowserEngineClient { sharedBenchClient ??= new BrowserEngineClient(); return sharedBenchClient; } function manifestUrl(): string { return new URL(`${import.meta.env.BASE_URL}${MANIFEST_PATH}`, document.baseURI).href; } function stateDriftReferenceUrl(): string { return new URL( `${import.meta.env.BASE_URL}${STATE_DRIFT_REFERENCE_PATH}`, document.baseURI, ).href; } function errorMessage(error: unknown): string { if (error instanceof EngineClientError) return `${error.code}: ${error.message}`; return error instanceof Error ? error.message : String(error); } function isAbort(error: unknown, signal: AbortSignal): boolean { return signal.aborted || (error instanceof DOMException && error.name === 'AbortError') || (error instanceof EngineClientError && ['ABORTED', 'SHARD_DOWNLOAD_ABORTED'].includes(error.code)); } function throwIfAborted(signal: AbortSignal): void { if (signal.aborted) throw new DOMException('Benchmark stopped by operator.', 'AbortError'); } function parseOptionalInteger(value: string): number | null { if (value.trim() === '') return null; const parsed = Number(value); return Number.isSafeInteger(parsed) && parsed > 0 && parsed <= MAX_GLUE_INT ? parsed : null; } function formatDuration(value: number | null | undefined): string { if (value === null || value === undefined) return '—'; return value >= 1_000 ? `${(value / 1_000).toFixed(2)} s` : `${value.toFixed(1)} ms`; } function formatRate(value: number | null | undefined): string { return value === null || value === undefined ? '—' : `${value.toFixed(2)} t/s`; } function downloadReport(report: BenchmarkReport): void { const blob = new Blob([serializeBenchmarkReport(report)], { type: 'application/json' }); const url = URL.createObjectURL(blob); const anchor = document.createElement('a'); anchor.href = url; anchor.download = benchmarkReportFilename(report); document.body.append(anchor); anchor.click(); anchor.remove(); window.setTimeout(() => URL.revokeObjectURL(url), 0); } export function BenchApp() { const [client] = useState(getBenchClient); const [manifest, setManifest] = useState(null); const [capabilities, setCapabilities] = useState(null); const [workloadId, setWorkloadId] = useState(DEFAULT_BENCHMARK_WORKLOAD_ID); const [modelId, setModelId] = useState(DEFAULT_MODEL); const [backend, setBackend] = useState('auto'); const [contextSize, setContextSize] = useState(4_096); const [maxTokens, setMaxTokens] = useState(64); const [nBatchInput, setNBatchInput] = useState(''); const [nUbatchInput, setNUbatchInput] = useState(''); const [flashMode, setFlashMode] = useState('off'); const [kvCacheType, setKvCacheType] = useState('f16'); const [wasmFlavor, setWasmFlavor] = useState('auto'); const [running, setRunning] = useState(false); const [progress, setProgress] = useState({ label: 'Waiting for an operator-initiated run', loadedBytes: 0, totalBytes: 0, }); const [report, setReport] = useState(null); const [output, setOutput] = useState(''); const [bootstrapError, setBootstrapError] = useState(null); const [runError, setRunError] = useState(null); const operationRef = useRef(null); useEffect(() => { let cancelled = false; const controller = new AbortController(); setProgress((current) => ({ ...current, label: 'Inspecting manifest and browser runtime' })); void Promise.all([ loadModelManifestV2(manifestUrl(), controller.signal), client.capabilities(), ]).then(([loadedManifest, loadedCapabilities]) => { if (cancelled) return; const initial = loadedManifest.models.find((model) => model.id === DEFAULT_MODEL) ?? loadedManifest.models[0]; if (!initial) throw new Error('The benchmark manifest contains no model tiers.'); setManifest(loadedManifest); setCapabilities(loadedCapabilities); setModelId(initial.id); setContextSize(initial.defaultContext); setProgress({ label: 'Ready · no model is loaded until Run benchmark is pressed', loadedBytes: 0, totalBytes: 0, }); }).catch((bootstrapError) => { if (cancelled || controller.signal.aborted) return; setBootstrapError(errorMessage(bootstrapError)); setProgress((current) => ({ ...current, label: 'Runtime inspection failed' })); }); return () => { cancelled = true; controller.abort(); }; }, [client]); const model = useMemo( () => manifest?.models.find((candidate) => candidate.id === modelId) ?? null, [manifest, modelId], ); const workload = BENCHMARK_WORKLOADS[workloadId]; const deviceGate = useMemo(() => { if (!model || !capabilities) return null; return evaluateModelGate(model, backend, capabilities.webgpu, { usageBytes: null, quotaBytes: null, persisted: capabilities.storage.persisted, }); }, [backend, capabilities, model]); const storageGate = useMemo(() => { if (!model || !capabilities) return null; return evaluateModelGate(model, backend, capabilities.webgpu, capabilities.storage); }, [backend, capabilities, model]); const contextPolicy = model ? evaluateModelContextPolicy(model, contextSize, { tuningScope: 'benchmark', requestedBackend: backend, flashMode, kvCacheType, }) : null; const contextLimit = contextPolicy?.limit ?? 4_096; const nBatch = parseOptionalInteger(nBatchInput); const nUbatch = parseOptionalInteger(nUbatchInput); const batchTuningError = nBatchInput.trim() !== '' && nBatch === null ? `nBatch must be an integer from 1 to ${MAX_GLUE_INT.toLocaleString()}.` : nUbatchInput.trim() !== '' && nUbatch === null ? `nUbatch must be an integer from 1 to ${MAX_GLUE_INT.toLocaleString()}.` : nBatch !== null && nUbatch !== null && nUbatch > nBatch ? 'nUbatch must not exceed nBatch.' : null; const runtimeTuningError = flashMode === 'off' && kvCacheType !== 'f16' ? 'Quantized KV requires Flash Attention auto.' : flashMode === 'auto' && backend !== 'webgpu' ? 'Flash Attention experiments require the explicit WebGPU backend.' : null; const wasmTuningError = wasmFlavor === 'jspi' && capabilities?.runtime.wasmFlavor === 'compat' ? 'JSPI is unavailable in this browser; use auto or compat.' : null; const workloadError = workload.requiredMaxTokens !== null && maxTokens !== workload.requiredMaxTokens ? `${workload.id} requires exactly ${workload.requiredMaxTokens.toLocaleString()} max tokens.` : null; const stateDriftReferenceError = workload.id === 'state-drift-1k-v1' && ( model?.id !== '27b' || backend !== 'webgpu' || contextSize !== 2_048 || nBatch !== 32 || nUbatch !== 16 ) ? 'state-drift-1k-v1 reference scoring requires 27B, explicit WebGPU, context 2,048, nBatch 32, and nUbatch 16.' : null; const configurationTuningError = batchTuningError ?? runtimeTuningError ?? wasmTuningError ?? workloadError ?? stateDriftReferenceError; const configurationValid = contextPolicy?.allowed === true && Number.isSafeInteger(maxTokens) && maxTokens >= 8 && maxTokens <= 1_024 && configurationTuningError === null; const canRun = Boolean(manifest && capabilities && model && deviceGate?.allowed && configurationValid); const clearPreviousRun = () => { setReport(null); setOutput(''); setRunError(null); if (manifest && capabilities) { setProgress({ label: 'Ready · settings changed; run again to produce matching evidence', loadedBytes: 0, totalBytes: 0, }); } }; const handleModelChange = (nextId: ModelTierId) => { if (running || !manifest) return; const next = manifest.models.find((candidate) => candidate.id === nextId); if (!next) return; setModelId(nextId); setContextSize(next.defaultContext); if (!next.cpuFallback && backend === 'wasm') setBackend('auto'); clearPreviousRun(); }; const runBenchmark = () => { if (!canRun || !manifest || !capabilities || !model || operationRef.current) return; const controller = new AbortController(); operationRef.current = controller; setRunning(true); setReport(null); setOutput(''); setRunError(null); const startedAt = new Date().toISOString(); const run = async () => { let storagePersistent = capabilities.storage.persisted; setProgress({ label: 'Requesting persistent browser storage', loadedBytes: 0, totalBytes: 0 }); try { const storage = await client.storagePersist(); storagePersistent = storage.persisted; setCapabilities((current) => current ? { ...current, storage } : current); } catch { // Persistence is a browser policy choice. The runtime storage gate remains authoritative. } throwIfAborted(controller.signal); await client.unload(); throwIfAborted(controller.signal); let coldCachedBytes: number | null = null; const loadParams = { manifest, modelId: model.id, backend, contextSize, benchmarkTuning: { ...(nBatch === null ? {} : { nBatch }), ...(nUbatch === null ? {} : { nUbatch }), flashMode, kvCacheType, wasmFlavor, }, }; const coldStarted = performance.now(); const coldLoad = await client.loadModel(loadParams, { signal: controller.signal, onProgress: (event) => { if (event.phase === 'download' && coldCachedBytes === null) { coldCachedBytes = event.loadedBytes; } const shard = event.shardIndex === null ? '' : ` · shard ${event.shardIndex + 1}/${event.shardCount}`; setProgress({ label: `Cold load · ${event.phase}${shard}`, loadedBytes: event.loadedBytes, totalBytes: event.totalBytes, }); }, }); const coldLoadMs = performance.now() - coldStarted; throwIfAborted(controller.signal); setProgress({ label: 'Unloading model before verified-cache reload', loadedBytes: 0, totalBytes: 0 }); await client.unload(); throwIfAborted(controller.signal); const warmStarted = performance.now(); const warmLoad = await client.loadModel(loadParams, { signal: controller.signal, onProgress: (event) => { const shard = event.shardIndex === null ? '' : ` · shard ${event.shardIndex + 1}/${event.shardCount}`; setProgress({ label: `Warm load · ${event.phase}${shard}`, loadedBytes: event.loadedBytes, totalBytes: event.totalBytes, }); }, }); const warmLoadMs = performance.now() - warmStarted; if (coldLoad.backend !== warmLoad.backend) { throw new Error(`Backend changed between cold and warm loads (${coldLoad.backend} → ${warmLoad.backend}).`); } if (coldLoad.tuning.observed.wasmFlavor !== warmLoad.tuning.observed.wasmFlavor) { throw new Error( `WASM flavor changed between cold and warm loads (${coldLoad.tuning.observed.wasmFlavor} → ${warmLoad.tuning.observed.wasmFlavor}).`, ); } throwIfAborted(controller.signal); let firstTokenAt: number | null = null; let streamedTokenEvents = 0; const generationStarted = performance.now(); const generationRequest = { messages: workload.messages.map((message) => ({ ...message })), maxTokens, temperature: 0, topK: 1, topP: 1, seed: 42, toolChoice: 'none', cachePrompt: false, returnTokenIds: true, } satisfies GenerateParams; setProgress({ label: `Generating fixed workload · ${workload.id}`, loadedBytes: 0, totalBytes: maxTokens }); const generationResult = await client.generate(generationRequest, { signal: controller.signal, onToken: (text, reasoningDelta) => { if (!text && !reasoningDelta) return; if (firstTokenAt === null) firstTokenAt = performance.now(); streamedTokenEvents += 1; setProgress({ label: `Generating fixed prompt · ${streamedTokenEvents} stream event${streamedTokenEvents === 1 ? '' : 's'}`, loadedBytes: streamedTokenEvents, totalBytes: maxTokens, }); }, }); const generationCompleted = performance.now(); throwIfAborted(controller.signal); let teacherForcedReferenceScore: TeacherForcedReferenceScoreEvidence | null = null; if (workload.id === 'state-drift-1k-v1') { setProgress({ label: 'Loading and validating pinned CPU reference', loadedBytes: 0, totalBytes: 1, }); const fixtureResponse = await fetch(stateDriftReferenceUrl(), { signal: controller.signal, cache: 'no-cache', }); if (!fixtureResponse.ok) { throw new Error( `CPU reference fixture request failed with HTTP ${fixtureResponse.status}.`, ); } const fixture = validateStateDriftReferenceFixture( await fixtureResponse.json() as unknown, { model, capabilities, loadResult: warmLoad }, ); throwIfAborted(controller.signal); setProgress({ label: 'Teacher-forcing the 1,024-token CPU reference', loadedBytes: 0, totalBytes: fixture.referenceTokenIds.length, }); const score = await client.scoreSequence({ promptTokenIds: fixture.promptTokenIds, referenceTokenIds: fixture.referenceTokenIds, topK: 5, }, { signal: controller.signal }); teacherForcedReferenceScore = { fixture, score }; throwIfAborted(controller.signal); } const backendReport = await client.backendReport(); throwIfAborted(controller.signal); const nextReport = buildBenchmarkReport({ startedAt, completedAt: new Date().toISOString(), manifest, model, capabilities, requestedBackend: backend, contextSize: warmLoad.context.size, workloadId, generationRequest, coldLoadMs, warmLoadMs, coldCachedBytes, storagePersistent, loadResult: warmLoad, generationResult, generationElapsedMs: generationCompleted - generationStarted, timeToFirstTokenMs: firstTokenAt === null ? null : firstTokenAt - generationStarted, streamedTokenEvents, backendReport, teacherForcedReferenceScore, }); setReport(nextReport); setOutput(generationResult.text || generationResult.reasoningText); setProgress({ label: 'Benchmark complete · JSON report is ready', loadedBytes: 1, totalBytes: 1, }); }; void run().catch((runError) => { if (isAbort(runError, controller.signal)) { setProgress({ label: 'Benchmark stopped by operator', loadedBytes: 0, totalBytes: 0 }); return; } setRunError(errorMessage(runError)); setProgress({ label: 'Benchmark failed safely · no report exported', loadedBytes: 0, totalBytes: 0 }); }).finally(() => { if (operationRef.current === controller) operationRef.current = null; setRunning(false); }); }; const progressPercent = progress.totalBytes > 0 ? Math.min(100, Math.max(0, progress.loadedBytes / progress.totalBytes * 100)) : 0; const fullGateStorageOnly = storageGate && !storageGate.allowed && deviceGate?.allowed; return (
Nothing runs on arrival One click performs two loads, one fixed prompt, and no network call beyond model assets.
Open chat

Phase 5 · operator evidence

Measure the whole growth cycle.

Cold load includes cache inspection and any missing download. Warm load repeats initialization after unload with verified shards retained. Generation uses one public, fixed prompt.

{progress.label}{progress.totalBytes > 1 ? `${Math.round(progressPercent)}%` : ''}
{bootstrapError &&
Runtime inspection failed{bootstrapError}
} {runError &&
Run rejected{runError}
}

Core sample

Four boundaries, one run

{report && }
Cold load {formatDuration(report?.load.cold.durationMs)} {report ? `${report.load.cold.cacheState} at start` : 'first boundary'}
Warm load {formatDuration(report?.load.warm.durationMs)} verified cache retained
TTFT {formatDuration(report?.generation.timeToFirstTokenMs)} request to first stream
Decode {formatRate(report?.generation.decodeTokensPerSecond)} {report ? `${report.generation.completionTokens} completion tokens` : 'engine timing'}

Throughput

Prompt
{formatRate(report?.generation.promptTokensPerSecond)}
Decode
{formatRate(report?.generation.decodeTokensPerSecond)}
Token count
{report?.generation.completionTokens ?? '—'}
Finish
{report?.generation.finishReason ?? '—'}
CPU-ref PPL
{report?.generation.teacherForcedReferenceScore?.score.summary.perplexity.toFixed(4) ?? '—'}
CPU-ref mean NLL
{report?.generation.teacherForcedReferenceScore?.score.summary.meanNll.toFixed(6) ?? '—'}

Graph tripwire

Graph splits
{report?.execution.graphSplits ?? '—'}
CPU ops
{report?.execution.opsOnCpu ?? '—'}
GPU layers
{report?.execution.gpuLayers ? `${report.execution.gpuLayers.offloaded}/${report.execution.gpuLayers.total}` : '—'}
Backend
{report?.execution.selectedBackend.toUpperCase() ?? '—'}

Runtime pins

Device
{report?.execution.device ?? capabilities?.webgpu.name ?? 'Inspecting…'}
wllama
{report?.runtime.wllamaRevision.slice(0, 12) ?? '—'}
llama.cpp
{report?.runtime.llamaCppRevision.slice(0, 12) ?? '—'}
Weights
{report?.runtime.manifestRevision?.slice(0, 12) ?? '—'}
WASM
{report?.runtime.wasmFlavor.toUpperCase() ?? '—'}

Native tuning

Applied
{report ? (report.load.tuning.applied ? 'yes' : 'no') : '—'}
Flash
{report ? `${report.load.tuning.requested.flashMode} → ${String(report.load.tuning.observed.flashAttention)}` : '—'}
K/V
{report ? `${report.load.tuning.observed.cacheTypeK}/${report.load.tuning.observed.cacheTypeV}` : '—'}
KV bytes
{report?.load.tuning.observed.kvBufferBytes === null || report?.load.tuning.observed.kvBufferBytes === undefined ? '—' : formatBytes(report.load.tuning.observed.kvBufferBytes)}

Fixed prompt

{workload.id}

{workload.messages.map((message) => message.content ?? '').join('\n')}

Generated output · visible text, or reasoning trace

{output || 'Run the benchmark to inspect generated output.'}

); }