src/bench/report.test.ts

import { describe, expect, it } from 'vitest';
import type {
  BackendReport,
  EngineCapabilities,
  GenerateParams,
  LoadModelResult,
  ManifestModelV2,
  ModelManifestV2,
} from '../engine';
import {
  BENCHMARK_WORKLOADS,
  DEFAULT_BENCHMARK_WORKLOAD_ID,
  TOKEN_ID_CHECKPOINTS,
  benchmarkReportFilename,
  buildBenchmarkReport,
  classifyCacheState,
  serializeBenchmarkReport,
  type BenchmarkObservation,
  type TeacherForcedReferenceScoreEvidence,
} from './report';

const JSPI_WASM_SHA256 = 'd'.repeat(64);
const COMPAT_WASM_SHA256 = 'c'.repeat(64);
const COMPAT_WORKER_SHA256 = 'b'.repeat(64);

const model = {
  id: '8b',
  displayName: 'Bonsai 8B',
  architecture: 'qwen3next',
  source: {
    repo: 'prism-ml/Bonsai-8B-gguf',
    revision: '0123456789abcdef0123456789abcdef01234567',
    file: 'Bonsai-8B-Q1_0.gguf',
    bytes: 1_000,
    sha256: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
  },
  files: [{
    path: '8b/Bonsai-8B-Q1_0.gguf',
    bytes: 1_000,
    sha256: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
  }],
  downloadBytes: 1_000,
} as ManifestModelV2;

const manifest = {
  repository: {
    id: 'WaveCut/Bonsai-web-GGUF',
    revision: '112ea7a1a6229bde132b176b9a72477a7ecfde64',
  },
} as unknown as ModelManifestV2;

const capabilities = {
  crossOriginIsolated: true,
  sharedArrayBuffer: true,
  hardwareConcurrency: 12,
  storage: { usageBytes: 1_000, quotaBytes: 10_000, persisted: true },
  webgpu: {
    available: true,
    name: 'Apple M4 Max',
    vendor: 'Apple',
    architecture: 'metal-3',
    device: null,
    description: 'Apple M4 Max',
    limits: { maxStorageBufferBindingSize: 1_073_741_824 },
    features: ['shader-f16'],
  },
  browser: {
    userAgent: 'Mozilla/5.0 Chrome/150.0.0.0 Safari/537.36',
    platform: 'MacIntel',
    language: 'en-GB',
  },
  runtime: {
    implementation: 'bonsai-wllama',
    wllamaRevision: '912c18b75d4358c1405a64646b8dbe43a205943b',
    llamaCppRevision: '00fa7cb284cbf133fc426733bd64238a3588a33e',
    patchSetSha256: 'f'.repeat(64),
    moduleSha256: 'e'.repeat(64),
    wasmFlavor: 'jspi',
    wasmSha256: JSPI_WASM_SHA256,
    compatWorkerSha256: null,
    tokenEmbeddingOnWebGPU: true,
    tensorPlacementOverrides: false,
  },
} satisfies EngineCapabilities;

const backendReport = {
  backends: ['WebGPU'],
  nGraphSplits: 1,
  opsOnCpu: 0,
  layersGpu: { offloaded: 49, total: 49 },
  flashAttention: false,
  cacheTypeK: 'f16',
  cacheTypeV: 'f16',
  webgpuKvBufferBytes: 56 * 1024 ** 2,
  webgpuTrace: [],
} satisfies BackendReport;

const loadResult = {
  modelId: '8b',
  backend: 'webgpu',
  gate: {
    allowed: true,
    requestedBackend: 'auto',
    selectedBackend: 'webgpu',
    reasons: [],
    warnings: [],
    requiredBytes: 1_000,
    availableStorageBytes: 9_000,
  },
  shardUrls: ['https://models.test/bonsai-8b.gguf'],
  context: {
    size: 4_096,
    trainingSize: 32_768,
    layerCount: 49,
    vocabularySize: 151_936,
    batchSize: 2_048,
    microBatchSize: 512,
  },
  tuning: {
    scope: 'release-defaults',
    requested: {
      nBatch: null,
      nUbatch: null,
      flashMode: 'off',
      cacheTypeK: 'f16',
      cacheTypeV: 'f16',
      wasmFlavor: 'auto',
    },
    observed: {
      nBatch: 2_048,
      nUbatch: 512,
      flashAttention: false,
      cacheTypeK: 'f16',
      cacheTypeV: 'f16',
      kvBufferBytes: 56 * 1024 ** 2,
      wasmFlavor: 'jspi',
      wasmSha256: JSPI_WASM_SHA256,
      compatWorkerSha256: null,
    },
    applied: true,
  },
  chatTemplate: {
    bytes: 1_024,
    hasThinkMarker: true,
    hasToolCallMarker: true,
    hasToolResponseMarker: true,
  },
  backendReport,
} satisfies LoadModelResult;

const fieldCoreGenerationRequest: GenerateParams = {
  messages: BENCHMARK_WORKLOADS[DEFAULT_BENCHMARK_WORKLOAD_ID].messages.map((message) => ({
    ...message,
  })),
  maxTokens: 64,
  temperature: 0,
  topP: 1,
  topK: 1,
  seed: 42,
  toolChoice: 'none',
  cachePrompt: false,
  returnTokenIds: true,
};

function sampledTokenTrace(tokenIds: readonly number[]) {
  return tokenIds.map((id) => ({
    selected: { id, logprob: -0.1 },
    topCandidates: [
      { id, logprob: -0.1 },
      { id: id + 10_001, logprob: -1 },
      { id: id + 10_002, logprob: -2 },
      { id: id + 10_003, logprob: -3 },
      { id: id + 10_004, logprob: -4 },
    ],
  }));
}

function teacherForcedReferenceScore(
  referenceTokenIds: readonly number[],
): TeacherForcedReferenceScoreEvidence {
  const entries = referenceTokenIds.map((id, index) => ({
    index,
    selectedReference: { id, logprob: -0.1 },
    naturalTop1: { id, logprob: -0.1 },
    topCandidates: [
      { id, logprob: -0.1 },
      { id: id + 10_001, logprob: -1 },
      { id: id + 10_002, logprob: -2 },
      { id: id + 10_003, logprob: -3 },
      { id: id + 10_004, logprob: -4 },
    ],
    referenceRankInTopCandidatesZeroBased: 0,
    top1Top2Margin: 0.9,
  }));
  return {
    fixture: {
      schemaVersion: 1,
      kind: 'bonsai-state-drift-reference',
      workload: {
        id: 'state-drift-1k-v1',
        messages: BENCHMARK_WORKLOADS['state-drift-1k-v1'].messages.map((message) => ({
          role: 'user',
          content: message.content ?? '',
        })),
      },
      provenance: {
        sourceEvidence: 'results/space-model/fixture.json',
        engineRevision: capabilities.runtime.llamaCppRevision,
        nativeBinary: { bytes: 1, sha256: '1'.repeat(64) },
        model: { file: model.source.file, bytes: model.source.bytes, sha256: model.source.sha256 },
        renderedPromptSha256: '2'.repeat(64),
        execution: { backend: 'cpu', contextSize: 4_096, batchSize: 2_048, microBatchSize: 512 },
      },
      tokenEncoding: 'uint32-le',
      promptTokenIds: Array.from({ length: 38 }, (_, index) => index + 1),
      promptTokenIdsSha256: '3'.repeat(64),
      referenceTokenIds: [...referenceTokenIds],
      referenceTokenIdsSha256: '4'.repeat(64),
      checkpointPrefixes: TOKEN_ID_CHECKPOINTS.map((tokens) => ({
        tokens,
        sha256: '5'.repeat(64),
      })),
    },
    score: {
      method: {
        promptMode: 'raw-token-id-prefix',
        maxTokensPerStep: 1,
        temperature: 0,
        topK: 1,
        reportedTopLogprobs: 5,
        logitBias: 1_000,
        cachePromptFirst: false,
        cachePromptSubsequent: true,
      },
      entries,
      summary: { tokenCount: 1_024, meanNll: 0.1, perplexity: Math.exp(0.1) },
    },
  };
}

function observation(overrides: Partial<BenchmarkObservation> = {}): BenchmarkObservation {
  return {
    startedAt: '2026-07-15T10:00:00.000Z',
    completedAt: '2026-07-15T10:00:10.123Z',
    manifest,
    model,
    capabilities,
    requestedBackend: 'auto',
    contextSize: 4_096,
    workloadId: DEFAULT_BENCHMARK_WORKLOAD_ID,
    generationRequest: {
      ...fieldCoreGenerationRequest,
      messages: fieldCoreGenerationRequest.messages.map((message) => ({ ...message })),
    },
    coldLoadMs: 1_234.56,
    warmLoadMs: 234.54,
    coldCachedBytes: 250,
    storagePersistent: true,
    loadResult,
    generationResult: {
      text: 'fixture',
      reasoningText: '',
      tokenIds: [101, 102, 103, 104, 105, 106, 107, 108],
      tokenTrace: sampledTokenTrace([101, 102, 103, 104, 105, 106, 107, 108]),
      finishReason: 'stop',
      toolCalls: [],
      usage: { promptTokens: 20, completionTokens: 8, totalTokens: 28 },
      timings: { promptTokensPerSecond: 123.456, predictedTokensPerSecond: 45.678 },
    },
    generationElapsedMs: 900.04,
    timeToFirstTokenMs: 123.45,
    streamedTokenEvents: 5,
    backendReport,
    ...overrides,
  };
}

function longContextObservation(): BenchmarkObservation {
  const longContextBackendReport = {
    ...backendReport,
    layersGpu: { offloaded: 65, total: 65 },
    flashAttention: true,
    cacheTypeK: 'q4_0',
    cacheTypeV: 'q4_0',
    webgpuKvBufferBytes: 155_713_536,
  } satisfies BackendReport;
  const longContextLoadResult = {
    ...loadResult,
    modelId: '27b',
    gate: {
      ...loadResult.gate,
      requestedBackend: 'webgpu',
      selectedBackend: 'webgpu',
    },
    context: { ...loadResult.context, size: 8_448, layerCount: 65 },
    tuning: {
      scope: 'benchmark',
      requested: {
        nBatch: 32,
        nUbatch: 16,
        flashMode: 'auto',
        cacheTypeK: 'q4_0',
        cacheTypeV: 'q4_0',
        wasmFlavor: 'auto',
      },
      observed: {
        nBatch: 32,
        nUbatch: 16,
        flashAttention: true,
        cacheTypeK: 'q4_0',
        cacheTypeV: 'q4_0',
        kvBufferBytes: 155_713_536,
        wasmFlavor: 'jspi',
        wasmSha256: JSPI_WASM_SHA256,
        compatWorkerSha256: null,
      },
      applied: true,
    },
    backendReport: longContextBackendReport,
  } satisfies LoadModelResult;

  return observation({
    model: {
      ...model,
      id: '27b',
      displayName: 'Bonsai 27B',
      architecture: 'qwen35',
    } as ManifestModelV2,
    requestedBackend: 'webgpu',
    contextSize: 8_448,
    workloadId: 'context-prefill-8k-v1',
    generationRequest: {
      messages: BENCHMARK_WORKLOADS['context-prefill-8k-v1'].messages.map((message) => ({ ...message })),
      maxTokens: 8,
      temperature: 0,
      topP: 1,
      topK: 1,
      seed: 42,
      toolChoice: 'none',
      cachePrompt: false,
      returnTokenIds: true,
    },
    loadResult: longContextLoadResult,
    backendReport: longContextBackendReport,
    streamedTokenEvents: 8,
    generationResult: {
      text: 'fixture',
      reasoningText: '',
      tokenIds: [201, 202, 203, 204, 205, 206, 207, 208],
      tokenTrace: sampledTokenTrace([201, 202, 203, 204, 205, 206, 207, 208]),
      finishReason: 'length',
      toolCalls: [],
      usage: { promptTokens: 8_314, completionTokens: 8, totalTokens: 8_322 },
      timings: { promptTokensPerSecond: 20, predictedTokensPerSecond: 30 },
    },
  });
}

describe('benchmark report', () => {
  it('classifies the shard cache observed at the first load boundary', () => {
    expect(classifyCacheState(null, 1_000)).toBe('unknown');
    expect(classifyCacheState(0, 1_000)).toBe('empty');
    expect(classifyCacheState(1, 1_000)).toBe('partial');
    expect(classifyCacheState(1_000, 1_000)).toBe('cached');
  });

  it('builds a shareable report with pinned runtime evidence and fixed-safe policy', () => {
    const report = buildBenchmarkReport(observation());

    expect(report.schemaVersion).toBe(3);
    expect(report.model).toMatchObject({
      source: model.source,
      shards: model.files,
    });
    expect(report.load).toMatchObject({
      tuning: {
        scope: 'release-defaults',
        requested: {
          nBatch: null,
          nUbatch: null,
          flashMode: 'off',
          cacheTypeK: 'f16',
          cacheTypeV: 'f16',
          wasmFlavor: 'auto',
        },
        observed: {
          nBatch: 2_048,
          nUbatch: 512,
          flashAttention: false,
          cacheTypeK: 'f16',
          cacheTypeV: 'f16',
          kvBufferBytes: 56 * 1024 ** 2,
          wasmFlavor: 'jspi',
          wasmSha256: JSPI_WASM_SHA256,
          compatWorkerSha256: null,
        },
        applied: true,
      },
      cold: { durationMs: 1_234.6, cacheState: 'partial' },
      warm: { durationMs: 234.5 },
    });
    expect(report.generation).toMatchObject({
      workload: {
        id: 'field-core-v1',
        messages: BENCHMARK_WORKLOADS['field-core-v1'].messages,
      },
      sampling: {
        maxTokens: 64,
        temperature: 0,
        topP: 1,
        topK: 1,
        minP: null,
        seed: 42,
        toolChoice: 'none',
        cachePrompt: false,
      },
      tokenTrace: {
        returnTokenIds: true,
        logprobs: true,
        topLogprobs: 5,
        topAlternatives: 4,
        encoding: 'uint32-le',
        tokenIds: [101, 102, 103, 104, 105, 106, 107, 108],
        entries: sampledTokenTrace([101, 102, 103, 104, 105, 106, 107, 108]),
        checkpointPrefixes: [],
      },
      rawText: 'fixture',
      rawReasoningText: '',
      promptTokensPerSecond: 123.46,
      decodeTokensPerSecond: 45.68,
      engineCompletionTokens: 8,
      streamedTokenEvents: 5,
      completionTokens: 8,
      tokenCountSource: 'token-id-trace',
    });
    expect(report.execution).toMatchObject({
      selectedBackend: 'webgpu',
      device: 'Apple M4 Max',
      graphSplits: 1,
      opsOnCpu: 0,
    });
    expect(report.runtime.manifestRevision).toBe('112ea7a1a6229bde132b176b9a72477a7ecfde64');
    expect(report.runtime).toMatchObject({
      patchSetSha256: 'f'.repeat(64),
      moduleSha256: 'e'.repeat(64),
      wasmFlavor: 'jspi',
      wasmSha256: JSPI_WASM_SHA256,
      compatWorkerSha256: null,
    });
    expect(report.environment).toEqual({
      browser: {
        userAgent: 'Mozilla/5.0 Chrome/150.0.0.0 Safari/537.36',
        platform: 'MacIntel',
        language: 'en-GB',
      },
      webgpu: {
        available: true,
        name: 'Apple M4 Max',
        vendor: 'Apple',
        architecture: 'metal-3',
        device: null,
        description: 'Apple M4 Max',
        limits: { maxStorageBufferBindingSize: 1_073_741_824 },
        features: ['shader-f16'],
      },
    });
    expect(report.releasePolicy).toEqual({
      flashAttention: {
        enabled: false,
        operatorConfigurable: false,
        status: 'fixed-off-unvalidated-for-release',
      },
      kvCache: {
        key: 'f16',
        value: 'f16',
        operatorConfigurable: false,
        status: 'fixed-release-default-alternatives-unvalidated',
      },
    });
  });

  it('records requested and observed experimental tuning without relabelling it as release policy', () => {
    const experimentalLoadResult: LoadModelResult = {
      ...loadResult,
      context: {
        ...loadResult.context,
        batchSize: 512,
        microBatchSize: 128,
      },
      tuning: {
        scope: 'benchmark',
        requested: {
          nBatch: 512,
          nUbatch: 128,
          flashMode: 'auto',
          cacheTypeK: 'q8_0',
          cacheTypeV: 'q8_0',
          wasmFlavor: 'compat',
        },
        observed: {
          nBatch: 512,
          nUbatch: 128,
          flashAttention: true,
          cacheTypeK: 'q8_0',
          cacheTypeV: 'q8_0',
          kvBufferBytes: 28 * 1024 ** 2,
          wasmFlavor: 'compat',
          wasmSha256: COMPAT_WASM_SHA256,
          compatWorkerSha256: COMPAT_WORKER_SHA256,
        },
        applied: true,
      },
    };
    const report = buildBenchmarkReport(observation({ loadResult: experimentalLoadResult }));

    expect(report.load.tuning).toEqual({
      scope: 'benchmark',
      requested: {
        nBatch: 512,
        nUbatch: 128,
        flashMode: 'auto',
        cacheTypeK: 'q8_0',
        cacheTypeV: 'q8_0',
        wasmFlavor: 'compat',
      },
      observed: {
        nBatch: 512,
        nUbatch: 128,
        flashAttention: true,
        cacheTypeK: 'q8_0',
        cacheTypeV: 'q8_0',
        kvBufferBytes: 28 * 1024 ** 2,
        wasmFlavor: 'compat',
        wasmSha256: COMPAT_WASM_SHA256,
        compatWorkerSha256: COMPAT_WORKER_SHA256,
      },
      applied: true,
    });
    expect(report.runtime).toMatchObject({
      wasmFlavor: 'compat',
      wasmSha256: COMPAT_WASM_SHA256,
      compatWorkerSha256: COMPAT_WORKER_SHA256,
    });
  });

  it('exports reasoning-only output separately from visible content', () => {
    const input = observation();
    input.generationResult.text = '';
    input.generationResult.reasoningText = 'reasoning-only fixture';

    const report = buildBenchmarkReport(input);

    expect(report.generation.rawText).toBe('');
    expect(report.generation.rawReasoningText).toBe('reasoning-only fixture');
  });

  it('uses the exact token-id trace when final engine usage is present but incomplete', () => {
    const report = buildBenchmarkReport(observation({
      streamedTokenEvents: 64,
      generationResult: {
        text: 'fixture',
        reasoningText: '',
        tokenIds: Array.from({ length: 64 }, (_, index) => index + 1),
        tokenTrace: sampledTokenTrace(Array.from({ length: 64 }, (_, index) => index + 1)),
        finishReason: 'length',
        toolCalls: [],
        usage: { promptTokens: 5, completionTokens: 0, totalTokens: 5 },
        timings: { promptTokensPerSecond: 20, predictedTokensPerSecond: 30 },
      },
    }));

    expect(report.generation).toMatchObject({
      engineCompletionTokens: 0,
      streamedTokenEvents: 64,
      completionTokens: 64,
      engineTotalTokens: 5,
      totalTokens: 69,
      tokenCountSource: 'token-id-trace',
    });
    expect(report.generation.tokenTrace.checkpointPrefixes).toEqual([
      { tokens: 64, sha256: '0c8f462927e331f28e3f1a6d342957cd27118febc309bd3b2f646e2dfbaeec32' },
    ]);
  });

  it('uses the trace without emitting non-finite JSON metrics', () => {
    const report = buildBenchmarkReport(observation({
      generationElapsedMs: Number.NaN,
      timeToFirstTokenMs: Number.POSITIVE_INFINITY,
      generationResult: {
        text: 'fixture',
        reasoningText: '',
        tokenIds: [1, 2, 3, 4, 5],
        tokenTrace: sampledTokenTrace([1, 2, 3, 4, 5]),
        finishReason: 'length',
        toolCalls: [],
        usage: null,
        timings: null,
      },
    }));

    expect(report.generation).toMatchObject({
      elapsedMs: null,
      timeToFirstTokenMs: null,
      decodeTokensPerSecond: null,
      completionTokens: 5,
      tokenCountSource: 'token-id-trace',
    });
    expect(serializeBenchmarkReport(report)).not.toContain('NaN');
    expect(serializeBenchmarkReport(report)).toMatch(/\n$/);
  });

  it('records all locked state-drift checkpoints from the exact token-id prefixes', () => {
    const tokenIds = Array.from({ length: 1_024 }, (_, index) => index + 1);
    const report = buildBenchmarkReport(observation({
      workloadId: 'state-drift-1k-v1',
      generationRequest: {
        messages: BENCHMARK_WORKLOADS['state-drift-1k-v1'].messages.map((message) => ({ ...message })),
        maxTokens: 1_024,
        temperature: 0,
        topP: 1,
        topK: 1,
        seed: 42,
        toolChoice: 'none',
        cachePrompt: false,
        returnTokenIds: true,
      },
      streamedTokenEvents: 1_024,
      generationResult: {
        text: '1, 2, 3, …',
        reasoningText: '',
        tokenIds,
        tokenTrace: sampledTokenTrace(tokenIds),
        finishReason: 'length',
        toolCalls: [],
        usage: { promptTokens: 20, completionTokens: 1_024, totalTokens: 1_044 },
        timings: { promptTokensPerSecond: 20, predictedTokensPerSecond: 30 },
      },
      teacherForcedReferenceScore: teacherForcedReferenceScore(tokenIds),
    }));

    expect(report.generation.workload).toEqual({
      id: 'state-drift-1k-v1',
      messages: BENCHMARK_WORKLOADS['state-drift-1k-v1'].messages,
    });
    expect(report.generation.rawText).toBe('1, 2, 3, …');
    expect(report.generation.rawReasoningText).toBe('');
    expect(report.generation.tokenTrace.tokenIds).toEqual(tokenIds);
    expect(report.generation.tokenTrace.entries).toHaveLength(1_024);
    expect(report.generation.teacherForcedReferenceScore).toMatchObject({
      fixture: { kind: 'bonsai-state-drift-reference' },
      score: { summary: { tokenCount: 1_024, meanNll: 0.1 } },
    });
    expect(report.generation.tokenTrace.checkpointPrefixes).toEqual([
      { tokens: 64, sha256: '0c8f462927e331f28e3f1a6d342957cd27118febc309bd3b2f646e2dfbaeec32' },
      { tokens: 128, sha256: '24f9ac547baae524ba0ea5220692d48f7526cdb1df5e99edcbb1f32239a8d5f5' },
      { tokens: 256, sha256: '80fa0f6d1caca9aad2b012051399b33bcd1976b145f3f3eea0f7ba10637761b0' },
      { tokens: 512, sha256: '8ab9b2cf36ec9e9d68711df73334731d2fee0552f5d95d76538b1d2cdcefd564' },
      { tokens: 768, sha256: '23905fc64dc47867f49672959cddf676ca257967f5786eefaf8583ddf7ddf3e9' },
      { tokens: 1_024, sha256: '6b8b6bd30ff821daf5db90cc071525f5696e366efd1aeb30d0bf60f785a3c26d' },
    ]);
  });

  it('exports null teacher-forced scoring for non-state-drift workloads', () => {
    expect(buildBenchmarkReport(observation()).generation.teacherForcedReferenceScore).toBeNull();
  });

  it('rejects incomplete state-drift traces instead of exporting partial evidence', () => {
    expect(() => buildBenchmarkReport(observation({
      workloadId: 'state-drift-1k-v1',
      generationRequest: {
        messages: BENCHMARK_WORKLOADS['state-drift-1k-v1'].messages.map((message) => ({ ...message })),
        maxTokens: 1_024,
        temperature: 0,
        topP: 1,
        topK: 1,
        seed: 42,
        toolChoice: 'none',
        cachePrompt: false,
        returnTokenIds: true,
      },
      generationResult: {
        text: 'partial fixture',
        reasoningText: '',
        tokenIds: Array.from({ length: 1_023 }, (_, index) => index + 1),
        tokenTrace: sampledTokenTrace(Array.from({ length: 1_023 }, (_, index) => index + 1)),
        finishReason: 'length',
        toolCalls: [],
        usage: { promptTokens: 20, completionTokens: 1_023, totalTokens: 1_043 },
        timings: null,
      },
    }))).toThrow('requires exactly 1,024 sampled token ids');
  });

  it('rejects state-drift traces that stop before exhausting the locked budget', () => {
    const input = observation({
      workloadId: 'state-drift-1k-v1',
      generationRequest: {
        messages: BENCHMARK_WORKLOADS['state-drift-1k-v1'].messages.map((message) => ({ ...message })),
        maxTokens: 1_024,
        temperature: 0,
        topP: 1,
        topK: 1,
        seed: 42,
        toolChoice: 'none',
        cachePrompt: false,
        returnTokenIds: true,
      },
      generationResult: {
        text: 'early stop',
        reasoningText: '',
        tokenIds: Array.from({ length: 1_024 }, (_, index) => index + 1),
        tokenTrace: sampledTokenTrace(Array.from({ length: 1_024 }, (_, index) => index + 1)),
        finishReason: 'stop',
        toolCalls: [],
        usage: { promptTokens: 20, completionTokens: 1_024, totalTokens: 1_044 },
        timings: null,
      },
    });

    expect(() => buildBenchmarkReport(input)).toThrow('requires finishReason=length');
  });

  it('exports long-context evidence only after processing more than 8K prompt tokens', () => {
    const report = buildBenchmarkReport(longContextObservation());

    expect(BENCHMARK_WORKLOADS['context-prefill-8k-v1'].messages).toEqual([{
      role: 'user',
      content: `Continue the pattern.${' x'.repeat(8_300)}`,
    }]);
    expect(report).toMatchObject({
      model: { id: '27b', contextSize: 8_448 },
      generation: {
        workload: { id: 'context-prefill-8k-v1' },
        sampling: { maxTokens: 8 },
        promptTokens: 8_314,
        completionTokens: 8,
      },
      load: {
        tuning: {
          scope: 'benchmark',
          requested: { flashMode: 'auto', cacheTypeK: 'q4_0', cacheTypeV: 'q4_0' },
          observed: { flashAttention: true, cacheTypeK: 'q4_0', cacheTypeV: 'q4_0' },
          applied: true,
        },
      },
      execution: { requestedBackend: 'webgpu', selectedBackend: 'webgpu' },
    });
    expect(report.generation.tokenTrace.tokenIds).toHaveLength(8);
  });

  it('rejects long-context evidence at or below 8,192 engine-counted prompt tokens', () => {
    const input = longContextObservation();
    input.generationResult.usage = { promptTokens: 8_192, completionTokens: 8, totalTokens: 8_200 };

    expect(() => buildBenchmarkReport(input)).toThrow(
      'requires more than 8,192 engine-counted prompt tokens',
    );
  });

  it('rejects a long-context trace that stops before exhausting the 8-token budget', () => {
    const input = longContextObservation();
    input.generationResult.finishReason = 'stop';

    expect(() => buildBenchmarkReport(input)).toThrow('requires finishReason=length');
  });

  it('rejects long-context evidence with the wrong context or runtime tuning', () => {
    const wrongContext = longContextObservation();
    wrongContext.contextSize = 8_447;
    wrongContext.loadResult = {
      ...wrongContext.loadResult,
      context: { ...wrongContext.loadResult.context, size: 8_447 },
    };
    expect(() => buildBenchmarkReport(wrongContext)).toThrow(
      'requires Bonsai 27B at context 8,448',
    );

    const wrongTuning = longContextObservation();
    wrongTuning.loadResult = {
      ...wrongTuning.loadResult,
      tuning: {
        ...wrongTuning.loadResult.tuning,
        requested: {
          ...wrongTuning.loadResult.tuning.requested,
          cacheTypeK: 'q8_0',
          cacheTypeV: 'q8_0',
        },
        observed: {
          ...wrongTuning.loadResult.tuning.observed,
          cacheTypeK: 'q8_0',
          cacheTypeV: 'q8_0',
        },
      },
    };
    expect(() => buildBenchmarkReport(wrongTuning)).toThrow(
      'requires Bonsai 27B at context 8,448',
    );
  });

  it('rejects missing and malformed token-id traces', () => {
    const missingTrace = observation();
    missingTrace.generationResult.tokenIds = null;
    expect(() => buildBenchmarkReport(missingTrace)).toThrow('without a token-id trace');

    const malformedTrace = observation();
    malformedTrace.generationResult.tokenIds = [1, 2.5];
    expect(() => buildBenchmarkReport(malformedTrace)).toThrow('token id 1 is invalid');
  });

  it('rejects missing, mismatched, and unsorted sampled-token logprob traces', () => {
    const missingTrace = observation();
    missingTrace.generationResult.tokenTrace = null;
    expect(() => buildBenchmarkReport(missingTrace)).toThrow('without a sampled-token logprob trace');

    const mismatchedTrace = observation();
    mismatchedTrace.generationResult.tokenTrace?.pop();
    expect(() => buildBenchmarkReport(mismatchedTrace)).toThrow(
      'sampled-token trace length 7 does not match token-id length 8',
    );

    const unsortedTrace = observation();
    const candidates = unsortedTrace.generationResult.tokenTrace?.[0]?.topCandidates;
    if (!candidates) throw new Error('Unsorted trace fixture requires top candidates.');
    [candidates[0], candidates[1]] = [candidates[1]!, candidates[0]!];
    expect(() => buildBenchmarkReport(unsortedTrace)).toThrow('candidates are not sorted');
  });

  it('uses the model, selected backend, and UTC completion time in the export name', () => {
    const report = buildBenchmarkReport(observation());
    expect(benchmarkReportFilename(report)).toBe(
      'bonsai-bench-8b-webgpu-2026-07-15T10-00-10-123Z.json',
    );
  });
});