src/bench/report.ts

import { sha256 } from '@noble/hashes/sha2.js';
import { bytesToHex } from '@noble/hashes/utils.js';
import type {
  BackendReport,
  EngineCapabilities,
  EngineChatMessage,
  EngineSampledTokenTraceEntry,
  GenerateParams,
  GenerateResult,
  LoadModelResult,
  ManifestModelV2,
  ModelManifestV2,
  RequestedBackend,
  ScoreSequenceResult,
} from '../engine';
import type { StateDriftReferenceFixture } from './state-drift-reference';

export const BENCHMARK_WORKLOADS = {
  'field-core-v1': {
    id: 'field-core-v1',
    label: 'Field core · short',
    description: 'Stable throughput workload; completion length remains operator-selected.',
    requiredMaxTokens: null,
    messages: [{
      role: 'user',
      content: 'Describe how a tree survives one dry season in exactly eight short, factual sentences.',
    }],
  },
  'state-drift-1k-v1': {
    id: 'state-drift-1k-v1',
    label: 'State drift · 1K',
    description: 'P2.5 recurrent-state evidence; requires an exact 1,024-token trace.',
    requiredMaxTokens: 1_024,
    messages: [{
      role: 'user',
      content: 'Output consecutive integers starting at 1, separated only by a comma and one space. Continue without explanation until you reach 500.',
    }],
  },
  'context-prefill-8k-v1': {
    id: 'context-prefill-8k-v1',
    label: 'Context prefill · >8K',
    description: 'P2.5 long-context evidence; requires more than 8,192 engine-counted prompt tokens.',
    requiredMaxTokens: 8,
    messages: [{
      role: 'user',
      content: `Continue the pattern.${' x'.repeat(8_300)}`,
    }],
  },
} as const satisfies Record<string, {
  id: string;
  label: string;
  description: string;
  requiredMaxTokens: number | null;
  messages: readonly EngineChatMessage[];
}>;

export type BenchmarkWorkloadId = keyof typeof BENCHMARK_WORKLOADS;
export const DEFAULT_BENCHMARK_WORKLOAD_ID: BenchmarkWorkloadId = 'field-core-v1';
export const BENCHMARK_PROMPT_ID = DEFAULT_BENCHMARK_WORKLOAD_ID;
export const BENCHMARK_PROMPT = BENCHMARK_WORKLOADS[DEFAULT_BENCHMARK_WORKLOAD_ID].messages[0].content;
export const TOKEN_ID_CHECKPOINTS = [64, 128, 256, 512, 768, 1_024] as const;
export type TokenIdCheckpointSize = typeof TOKEN_ID_CHECKPOINTS[number];

export type CacheState = 'empty' | 'partial' | 'cached' | 'unknown';

export interface BenchmarkObservation {
  startedAt: string;
  completedAt: string;
  manifest: ModelManifestV2;
  model: ManifestModelV2;
  capabilities: EngineCapabilities;
  requestedBackend: RequestedBackend;
  contextSize: number;
  workloadId: BenchmarkWorkloadId;
  generationRequest: GenerateParams;
  coldLoadMs: number;
  warmLoadMs: number;
  coldCachedBytes: number | null;
  storagePersistent: boolean;
  loadResult: LoadModelResult;
  generationResult: GenerateResult;
  generationElapsedMs: number;
  timeToFirstTokenMs: number | null;
  streamedTokenEvents: number;
  backendReport: BackendReport;
  teacherForcedReferenceScore?: TeacherForcedReferenceScoreEvidence | null;
}

export interface TeacherForcedReferenceScoreEvidence {
  fixture: StateDriftReferenceFixture;
  score: ScoreSequenceResult;
}

export interface BenchmarkReport {
  schemaVersion: 3;
  kind: 'bonsai-browser-benchmark';
  startedAt: string;
  completedAt: string;
  model: {
    id: ManifestModelV2['id'];
    displayName: string;
    architecture: string;
    source: ManifestModelV2['source'];
    shards: Array<Pick<ManifestModelV2['files'][number], 'path' | 'bytes' | 'sha256'>>;
    downloadBytes: number;
    contextSize: number;
  };
  load: {
    tuning: LoadModelResult['tuning'];
    cold: {
      durationMs: number | null;
      cachedBytesAtStart: number | null;
      cacheState: CacheState;
      definition: string;
    };
    warm: {
      durationMs: number | null;
      definition: string;
    };
  };
  generation: {
    workload: {
      id: BenchmarkWorkloadId;
      messages: EngineChatMessage[];
    };
    sampling: {
      maxTokens: number;
      temperature: number;
      topP: number | null;
      topK: number | null;
      minP: number | null;
      seed: number;
      toolChoice: GenerateParams['toolChoice'] | null;
      cachePrompt: boolean;
    };
    tokenTrace: {
      returnTokenIds: true;
      logprobs: true;
      topLogprobs: 5;
      topAlternatives: 4;
      encoding: 'uint32-le';
      tokenIds: number[];
      entries: EngineSampledTokenTraceEntry[];
      checkpointPrefixes: Array<{
        tokens: TokenIdCheckpointSize;
        sha256: string;
      }>;
    };
    teacherForcedReferenceScore: TeacherForcedReferenceScoreEvidence | null;
    rawText: string;
    rawReasoningText: string;
    elapsedMs: number | null;
    timeToFirstTokenMs: number | null;
    promptTokensPerSecond: number | null;
    decodeTokensPerSecond: number | null;
    promptTokens: number | null;
    engineCompletionTokens: number | null;
    streamedTokenEvents: number;
    completionTokens: number;
    engineTotalTokens: number | null;
    totalTokens: number | null;
    tokenCountSource: 'token-id-trace';
    finishReason: GenerateResult['finishReason'];
  };
  execution: {
    requestedBackend: RequestedBackend;
    selectedBackend: LoadModelResult['backend'];
    device: string;
    graphSplits: number | null;
    opsOnCpu: number;
    gpuLayers: BackendReport['layersGpu'];
    confirmedBackends: string[];
  };
  runtime: {
    implementation: EngineCapabilities['runtime']['implementation'];
    wllamaRevision: string;
    llamaCppRevision: string;
    patchSetSha256: string;
    moduleSha256: string;
    wasmFlavor: EngineCapabilities['runtime']['wasmFlavor'];
    wasmSha256: string;
    compatWorkerSha256: string | null;
    manifestRepository: string;
    manifestRevision: string | null;
    crossOriginIsolated: boolean;
    sharedArrayBuffer: boolean;
    hardwareConcurrency: number;
    persistentStorage: boolean;
  };
  environment: {
    browser: EngineCapabilities['browser'];
    webgpu: EngineCapabilities['webgpu'];
  };
  releasePolicy: {
    flashAttention: {
      enabled: false;
      operatorConfigurable: false;
      status: 'fixed-off-unvalidated-for-release';
    };
    kvCache: {
      key: 'f16';
      value: 'f16';
      operatorConfigurable: false;
      status: 'fixed-release-default-alternatives-unvalidated';
    };
  };
}

function roundedNonNegative(value: number | null, digits: number): number | null {
  if (value === null || !Number.isFinite(value) || value < 0) return null;
  const scale = 10 ** digits;
  return Math.round(value * scale) / scale;
}

export function classifyCacheState(observedBytes: number | null, totalBytes: number): CacheState {
  if (observedBytes === null || !Number.isFinite(observedBytes) || totalBytes <= 0) return 'unknown';
  if (observedBytes <= 0) return 'empty';
  if (observedBytes >= totalBytes) return 'cached';
  return 'partial';
}

function cloneMessages(messages: readonly EngineChatMessage[]): EngineChatMessage[] {
  return messages.map((message) => message.role === 'assistant' && message.tool_calls
    ? {
        ...message,
        tool_calls: message.tool_calls.map((call) => ({
          ...call,
          function: { ...call.function },
        })),
      }
    : { ...message });
}

function checkpointPrefixSha256(tokenIds: readonly number[], tokens: number): string {
  const bytes = new Uint8Array(tokens * Uint32Array.BYTES_PER_ELEMENT);
  const view = new DataView(bytes.buffer);
  for (let index = 0; index < tokens; index += 1) {
    view.setUint32(index * Uint32Array.BYTES_PER_ELEMENT, tokenIds[index]!, true);
  }
  return bytesToHex(sha256(bytes));
}

function buildCheckpointPrefixes(tokenIds: readonly number[]): BenchmarkReport['generation']['tokenTrace']['checkpointPrefixes'] {
  return TOKEN_ID_CHECKPOINTS
    .filter((tokens) => tokens <= tokenIds.length)
    .map((tokens) => ({ tokens, sha256: checkpointPrefixSha256(tokenIds, tokens) }));
}

function cloneValidatedTokenTrace(
  value: GenerateResult['tokenTrace'],
  tokenIds: readonly number[],
): EngineSampledTokenTraceEntry[] {
  if (value === null) {
    throw new Error('Benchmark generation completed without a sampled-token logprob trace.');
  }
  if (value.length !== tokenIds.length) {
    throw new Error(
      `Benchmark sampled-token trace length ${value.length} does not match token-id length ${tokenIds.length}.`,
    );
  }
  return value.map((entry, index) => {
    const selected = { ...entry.selected };
    if (selected.id !== tokenIds[index]) {
      throw new Error(`Benchmark sampled-token trace entry ${index} does not match its token id.`);
    }
    if (typeof selected.logprob !== 'number' || !Number.isFinite(selected.logprob)) {
      throw new Error(`Benchmark sampled-token trace entry ${index} has an invalid selected logprob.`);
    }
    if (!Array.isArray(entry.topCandidates) || entry.topCandidates.length !== 5) {
      throw new Error(`Benchmark sampled-token trace entry ${index} must contain exactly five candidates.`);
    }
    const seenIds = new Set<number>();
    const topCandidates = entry.topCandidates.map((candidate, candidateIndex) => {
      if (!Number.isSafeInteger(candidate.id) || candidate.id < 0 || candidate.id > 0xffff_ffff) {
        throw new Error(
          `Benchmark sampled-token trace entry ${index} candidate ${candidateIndex} has an invalid id.`,
        );
      }
      if (seenIds.has(candidate.id)) {
        throw new Error(`Benchmark sampled-token trace entry ${index} has duplicate candidate ids.`);
      }
      seenIds.add(candidate.id);
      if (typeof candidate.logprob !== 'number' || !Number.isFinite(candidate.logprob)) {
        throw new Error(
          `Benchmark sampled-token trace entry ${index} candidate ${candidateIndex} has an invalid logprob.`,
        );
      }
      const previous = entry.topCandidates[candidateIndex - 1];
      if (previous && (
        previous.logprob < candidate.logprob
        || (previous.logprob === candidate.logprob && previous.id > candidate.id)
      )) {
        throw new Error(`Benchmark sampled-token trace entry ${index} candidates are not sorted.`);
      }
      return { ...candidate };
    });
    const selectedCandidate = topCandidates.find((candidate) => candidate.id === selected.id);
    if (!selectedCandidate || selectedCandidate.logprob !== selected.logprob) {
      throw new Error(
        `Benchmark sampled-token trace entry ${index} does not include its selected token and logprob.`,
      );
    }
    return { selected, topCandidates };
  });
}

function cloneTeacherForcedReferenceScore(
  value: TeacherForcedReferenceScoreEvidence | null | undefined,
  workloadId: BenchmarkWorkloadId,
): TeacherForcedReferenceScoreEvidence | null {
  if (workloadId !== 'state-drift-1k-v1') {
    if (value != null) {
      throw new Error('Teacher-forced reference scoring is only valid for state-drift-1k-v1.');
    }
    return null;
  }
  if (value == null) {
    throw new Error('state-drift-1k-v1 requires a complete teacher-forced CPU reference score.');
  }
  const { fixture, score } = value;
  if (
    fixture.schemaVersion !== 1
    || fixture.kind !== 'bonsai-state-drift-reference'
    || fixture.workload.id !== 'state-drift-1k-v1'
    || fixture.tokenEncoding !== 'uint32-le'
    || fixture.promptTokenIds.length !== 38
    || fixture.referenceTokenIds.length !== 1_024
  ) {
    throw new Error('state-drift-1k-v1 received an invalid CPU reference fixture.');
  }
  const expectedMethod: ScoreSequenceResult['method'] = {
    promptMode: 'raw-token-id-prefix',
    maxTokensPerStep: 1,
    temperature: 0,
    topK: 1,
    reportedTopLogprobs: 5,
    logitBias: 1_000,
    cachePromptFirst: false,
    cachePromptSubsequent: true,
  };
  if (JSON.stringify(score.method) !== JSON.stringify(expectedMethod)) {
    throw new Error('state-drift-1k-v1 teacher-forced method metadata is not locked.');
  }
  if (score.entries.length !== 1_024 || score.summary.tokenCount !== 1_024) {
    throw new Error('state-drift-1k-v1 requires exactly 1,024 teacher-forced score entries.');
  }
  const entries = score.entries.map((entry, index) => {
    const referenceTokenId = fixture.referenceTokenIds[index];
    if (entry.index !== index || entry.selectedReference.id !== referenceTokenId) {
      throw new Error(`Teacher-forced score entry ${index} does not select its CPU reference token.`);
    }
    if (!Number.isFinite(entry.selectedReference.logprob)) {
      throw new Error(`Teacher-forced score entry ${index} has a non-finite reference logprob.`);
    }
    if (!Array.isArray(entry.topCandidates) || entry.topCandidates.length !== 5) {
      throw new Error(`Teacher-forced score entry ${index} must contain five natural candidates.`);
    }
    const seenIds = new Set<number>();
    const topCandidates = entry.topCandidates.map((candidate, candidateIndex) => {
      if (
        !Number.isSafeInteger(candidate.id)
        || candidate.id < 0
        || !Number.isFinite(candidate.logprob)
        || seenIds.has(candidate.id)
      ) {
        throw new Error(`Teacher-forced score entry ${index} candidate ${candidateIndex} is invalid.`);
      }
      seenIds.add(candidate.id);
      const previous = entry.topCandidates[candidateIndex - 1];
      if (previous && (
        previous.logprob < candidate.logprob
        || (previous.logprob === candidate.logprob && previous.id > candidate.id)
      )) {
        throw new Error(`Teacher-forced score entry ${index} candidates are not sorted.`);
      }
      return { ...candidate };
    });
    if (
      entry.naturalTop1.id !== topCandidates[0]?.id
      || entry.naturalTop1.logprob !== topCandidates[0]?.logprob
    ) {
      throw new Error(`Teacher-forced score entry ${index} has inconsistent natural top-1 data.`);
    }
    const referenceRank = topCandidates.findIndex((candidate) => candidate.id === referenceTokenId);
    const normalizedReferenceRank = referenceRank === -1 ? null : referenceRank;
    if (entry.referenceRankInTopCandidatesZeroBased !== normalizedReferenceRank) {
      throw new Error(`Teacher-forced score entry ${index} has an invalid reference rank.`);
    }
    const margin = topCandidates[0]!.logprob - topCandidates[1]!.logprob;
    if (!Number.isFinite(entry.top1Top2Margin) || Math.abs(entry.top1Top2Margin - margin) > 1e-12) {
      throw new Error(`Teacher-forced score entry ${index} has an invalid top-1/top-2 margin.`);
    }
    return {
      index,
      selectedReference: { ...entry.selectedReference },
      naturalTop1: { ...entry.naturalTop1 },
      topCandidates,
      referenceRankInTopCandidatesZeroBased: normalizedReferenceRank,
      top1Top2Margin: entry.top1Top2Margin,
    };
  });
  const meanNll = -entries.reduce(
    (sum, entry) => sum + entry.selectedReference.logprob,
    0,
  ) / entries.length;
  const perplexity = Math.exp(meanNll);
  if (
    !Number.isFinite(score.summary.meanNll)
    || !Number.isFinite(score.summary.perplexity)
    || Math.abs(score.summary.meanNll - meanNll) > 1e-12
    || Math.abs(score.summary.perplexity - perplexity) > Math.max(1, perplexity) * 1e-12
  ) {
    throw new Error('Teacher-forced score summary does not match its reference logprobs.');
  }

  return {
    fixture: {
      schemaVersion: 1,
      kind: 'bonsai-state-drift-reference',
      workload: {
        id: 'state-drift-1k-v1',
        messages: fixture.workload.messages.map((message) => ({ ...message })),
      },
      provenance: {
        sourceEvidence: fixture.provenance.sourceEvidence,
        engineRevision: fixture.provenance.engineRevision,
        nativeBinary: { ...fixture.provenance.nativeBinary },
        model: { ...fixture.provenance.model },
        renderedPromptSha256: fixture.provenance.renderedPromptSha256,
        execution: { ...fixture.provenance.execution },
      },
      tokenEncoding: 'uint32-le',
      promptTokenIds: [...fixture.promptTokenIds],
      promptTokenIdsSha256: fixture.promptTokenIdsSha256,
      referenceTokenIds: [...fixture.referenceTokenIds],
      referenceTokenIdsSha256: fixture.referenceTokenIdsSha256,
      checkpointPrefixes: fixture.checkpointPrefixes.map((checkpoint) => ({ ...checkpoint })),
    },
    score: {
      method: { ...expectedMethod },
      entries,
      summary: {
        tokenCount: 1_024,
        meanNll: score.summary.meanNll,
        perplexity: score.summary.perplexity,
      },
    },
  };
}

export function buildBenchmarkReport(observation: BenchmarkObservation): BenchmarkReport {
  const { generationResult, capabilities, backendReport, loadResult } = observation;
  const workload = BENCHMARK_WORKLOADS[observation.workloadId];
  const generationRequest = observation.generationRequest;
  const maxTokens = generationRequest.maxTokens ?? 512;
  const temperature = generationRequest.temperature ?? 0;
  const topP = generationRequest.topP ?? null;
  const topK = generationRequest.topK ?? 1;
  const minP = generationRequest.minP ?? null;
  const seed = generationRequest.seed ?? 42;
  const toolChoice = generationRequest.toolChoice ?? null;
  const cachePrompt = generationRequest.cachePrompt ?? true;
  if (generationRequest.returnTokenIds !== true) {
    throw new Error('Benchmark reports require returnTokenIds=true.');
  }
  if (JSON.stringify(generationRequest.messages) !== JSON.stringify(workload.messages)) {
    throw new Error(`Benchmark messages do not match workload ${workload.id}.`);
  }
  if (generationResult.tokenIds === null) {
    throw new Error('Benchmark generation completed without a token-id trace.');
  }
  const tokenIds = [...generationResult.tokenIds];
  for (const [index, tokenId] of tokenIds.entries()) {
    if (!Number.isSafeInteger(tokenId) || tokenId < 0 || tokenId > 0xffff_ffff) {
      throw new Error(`Benchmark token id ${index} is invalid.`);
    }
  }
  const tokenTrace = cloneValidatedTokenTrace(generationResult.tokenTrace, tokenIds);
  if (typeof generationResult.reasoningText !== 'string') {
    throw new Error('Benchmark generation completed without a reasoning-text channel.');
  }
  const checkpointPrefixes = buildCheckpointPrefixes(tokenIds);
  if (workload.requiredMaxTokens !== null) {
    if (
      maxTokens !== workload.requiredMaxTokens
      || temperature !== 0
      || topP !== 1
      || topK !== 1
      || minP !== null
      || seed !== 42
      || toolChoice !== 'none'
      || cachePrompt !== false
    ) {
      throw new Error(`${workload.id} requires its locked greedy sampling configuration.`);
    }
  }
  if (workload.id === 'state-drift-1k-v1') {
    if (tokenIds.length !== workload.requiredMaxTokens || checkpointPrefixes.length !== TOKEN_ID_CHECKPOINTS.length) {
      throw new Error(`${workload.id} requires exactly 1,024 sampled token ids and all checkpoint hashes.`);
    }
    if (generationResult.finishReason !== 'length') {
      throw new Error(`${workload.id} requires finishReason=length after exhausting its 1,024-token budget.`);
    }
  }
  const usage = generationResult.usage;
  const engineCompletionTokens = usage
    && Number.isFinite(usage.completionTokens)
    && usage.completionTokens >= 0
    ? usage.completionTokens
    : null;
  const streamedTokenEvents = Number.isFinite(observation.streamedTokenEvents)
    ? Math.floor(Math.max(0, observation.streamedTokenEvents))
    : 0;
  const completionTokens = tokenIds.length;
  const promptTokens = usage && Number.isFinite(usage.promptTokens) && usage.promptTokens >= 0
    ? Math.floor(usage.promptTokens)
    : null;
  if (workload.id === 'context-prefill-8k-v1') {
    const tuning = loadResult.tuning;
    const exactRuntimePolicy = observation.model.id === '27b'
      && loadResult.modelId === '27b'
      && observation.contextSize === 8_448
      && loadResult.context.size === 8_448
      && observation.requestedBackend === 'webgpu'
      && loadResult.backend === 'webgpu'
      && tuning.scope === 'benchmark'
      && tuning.requested.flashMode === 'auto'
      && tuning.requested.cacheTypeK === 'q4_0'
      && tuning.requested.cacheTypeV === 'q4_0'
      && tuning.observed.flashAttention === true
      && tuning.observed.cacheTypeK === 'q4_0'
      && tuning.observed.cacheTypeV === 'q4_0'
      && tuning.observed.kvBufferBytes !== null
      && tuning.observed.kvBufferBytes > 0
      && tuning.applied;
    if (!exactRuntimePolicy) {
      throw new Error(
        `${workload.id} requires Bonsai 27B at context 8,448 with explicit WebGPU benchmark Flash auto and applied Q4_0 K/V.`,
      );
    }
    if (tokenIds.length !== 8) {
      throw new Error(`${workload.id} requires exactly 8 sampled token ids.`);
    }
    if (generationResult.finishReason !== 'length') {
      throw new Error(`${workload.id} requires finishReason=length after exhausting its 8-token budget.`);
    }
    if (promptTokens === null || promptTokens <= 8_192) {
      throw new Error(`${workload.id} requires more than 8,192 engine-counted prompt tokens.`);
    }
  }
  const teacherForcedReferenceScore = cloneTeacherForcedReferenceScore(
    observation.teacherForcedReferenceScore,
    workload.id,
  );
  const engineTotalTokens = usage && Number.isFinite(usage.totalTokens) && usage.totalTokens >= 0
    ? Math.floor(usage.totalTokens)
    : null;
  const totalTokens = usage
    ? Math.max(engineTotalTokens ?? 0, (promptTokens ?? 0) + completionTokens)
    : null;
  const decodeRate = generationResult.timings?.predictedTokensPerSecond
    ?? (completionTokens > 0 && observation.generationElapsedMs > 0
      ? completionTokens / (observation.generationElapsedMs / 1000)
      : null);
  const device = loadResult.backend === 'webgpu'
    ? capabilities.webgpu.name || 'WebGPU adapter'
    : `CPU · ${capabilities.hardwareConcurrency} logical cores`;
  return {
    schemaVersion: 3,
    kind: 'bonsai-browser-benchmark',
    startedAt: observation.startedAt,
    completedAt: observation.completedAt,
    model: {
      id: observation.model.id,
      displayName: observation.model.displayName,
      architecture: observation.model.architecture,
      source: { ...observation.model.source },
      shards: observation.model.files.map(({ path, bytes, sha256 }) => ({ path, bytes, sha256 })),
      downloadBytes: observation.model.downloadBytes,
      contextSize: observation.contextSize,
    },
    load: {
      tuning: {
        scope: loadResult.tuning.scope,
        requested: { ...loadResult.tuning.requested },
        observed: { ...loadResult.tuning.observed },
        applied: loadResult.tuning.applied,
      },
      cold: {
        durationMs: roundedNonNegative(observation.coldLoadMs, 1),
        cachedBytesAtStart: roundedNonNegative(observation.coldCachedBytes, 0),
        cacheState: classifyCacheState(observation.coldCachedBytes, observation.model.downloadBytes),
        definition: 'First load in this benchmark run; previously verified browser shards may already be cached.',
      },
      warm: {
        durationMs: roundedNonNegative(observation.warmLoadMs, 1),
        definition: 'Second load after an explicit unload, with the verified shard cache retained.',
      },
    },
    generation: {
      workload: {
        id: workload.id,
        messages: cloneMessages(generationRequest.messages),
      },
      sampling: {
        maxTokens,
        temperature,
        topP,
        topK,
        minP,
        seed,
        toolChoice,
        cachePrompt,
      },
      tokenTrace: {
        returnTokenIds: true,
        logprobs: true,
        topLogprobs: 5,
        topAlternatives: 4,
        encoding: 'uint32-le',
        tokenIds,
        entries: tokenTrace,
        checkpointPrefixes,
      },
      teacherForcedReferenceScore,
      rawText: generationResult.text,
      rawReasoningText: generationResult.reasoningText,
      elapsedMs: roundedNonNegative(observation.generationElapsedMs, 1),
      timeToFirstTokenMs: roundedNonNegative(observation.timeToFirstTokenMs, 1),
      promptTokensPerSecond: roundedNonNegative(
        generationResult.timings?.promptTokensPerSecond ?? null,
        2,
      ),
      decodeTokensPerSecond: roundedNonNegative(decodeRate, 2),
      promptTokens,
      engineCompletionTokens,
      streamedTokenEvents,
      completionTokens,
      engineTotalTokens,
      totalTokens,
      tokenCountSource: 'token-id-trace',
      finishReason: generationResult.finishReason,
    },
    execution: {
      requestedBackend: observation.requestedBackend,
      selectedBackend: loadResult.backend,
      device,
      graphSplits: backendReport.nGraphSplits,
      opsOnCpu: backendReport.opsOnCpu,
      gpuLayers: backendReport.layersGpu,
      confirmedBackends: [...backendReport.backends],
    },
    runtime: {
      implementation: capabilities.runtime.implementation,
      wllamaRevision: capabilities.runtime.wllamaRevision,
      llamaCppRevision: capabilities.runtime.llamaCppRevision,
      patchSetSha256: capabilities.runtime.patchSetSha256,
      moduleSha256: capabilities.runtime.moduleSha256,
      wasmFlavor: loadResult.tuning.observed.wasmFlavor,
      wasmSha256: loadResult.tuning.observed.wasmSha256,
      compatWorkerSha256: loadResult.tuning.observed.compatWorkerSha256,
      manifestRepository: observation.manifest.repository.id,
      manifestRevision: observation.manifest.repository.revision,
      crossOriginIsolated: capabilities.crossOriginIsolated,
      sharedArrayBuffer: capabilities.sharedArrayBuffer,
      hardwareConcurrency: capabilities.hardwareConcurrency,
      persistentStorage: observation.storagePersistent,
    },
    environment: {
      browser: { ...capabilities.browser },
      webgpu: {
        ...capabilities.webgpu,
        limits: { ...capabilities.webgpu.limits },
        features: [...capabilities.webgpu.features],
      },
    },
    releasePolicy: {
      flashAttention: {
        enabled: false,
        operatorConfigurable: false,
        status: 'fixed-off-unvalidated-for-release',
      },
      kvCache: {
        key: 'f16',
        value: 'f16',
        operatorConfigurable: false,
        status: 'fixed-release-default-alternatives-unvalidated',
      },
    },
  };
}

export function serializeBenchmarkReport(report: BenchmarkReport): string {
  return `${JSON.stringify(report, null, 2)}\n`;
}

export function benchmarkReportFilename(report: BenchmarkReport): string {
  const timestamp = report.completedAt.replace(/[:.]/g, '-');
  return `bonsai-bench-${report.model.id}-${report.execution.selectedBackend}-${timestamp}.json`;
}