Valeriy Selitskiy commited on
Commit
36a170e
·
1 Parent(s): 432e8f2

Use 4096-token release defaults

Browse files
public/manifest/models.json CHANGED
@@ -218,7 +218,7 @@
218
  ],
219
  "downloadBytes": 3803453440,
220
  "contextLength": 262144,
221
- "defaultContext": 4000,
222
  "cpuFallback": false,
223
  "largestTensorBytes": 178790400,
224
  "requiredLimits": {
 
218
  ],
219
  "downloadBytes": 3803453440,
220
  "contextLength": 262144,
221
+ "defaultContext": 4096,
222
  "cpuFallback": false,
223
  "largestTensorBytes": 178790400,
224
  "requiredLimits": {
src/app/App.test.ts CHANGED
@@ -1,7 +1,9 @@
1
  import { describe, expect, it } from 'vitest';
 
2
  import { EngineClientError } from '../engine';
3
  import {
4
  DEFAULT_CONTEXT_SIZE,
 
5
  DEFAULT_MODEL_ID,
6
  modelArchitectureLabel,
7
  publishBackendReport,
@@ -39,8 +41,12 @@ describe('App release defaults', () => {
39
  expect(modelArchitectureLabel('qwen35')).toBe('Qwen3.5 hybrid');
40
  });
41
 
42
- it('reserves enough context for a 2K completion plus prompts and tool schemas', () => {
43
- expect(DEFAULT_CONTEXT_SIZE).toBe(4000);
 
 
 
 
44
  });
45
  });
46
 
 
1
  import { describe, expect, it } from 'vitest';
2
+ import releaseManifest from '../../public/manifest/models.json';
3
  import { EngineClientError } from '../engine';
4
  import {
5
  DEFAULT_CONTEXT_SIZE,
6
+ DEFAULT_MAX_TOKENS,
7
  DEFAULT_MODEL_ID,
8
  modelArchitectureLabel,
9
  publishBackendReport,
 
41
  expect(modelArchitectureLabel('qwen35')).toBe('Qwen3.5 hybrid');
42
  });
43
 
44
+ it('uses one 4K ceiling for context allocation and completion requests', () => {
45
+ expect(DEFAULT_CONTEXT_SIZE).toBe(4096);
46
+ expect(DEFAULT_MAX_TOKENS).toBe(4096);
47
+ expect(DEFAULT_CONTEXT_SIZE).toBe(DEFAULT_MAX_TOKENS);
48
+ expect(releaseManifest.models.find((model) => model.id === '27b')?.defaultContext)
49
+ .toBe(DEFAULT_CONTEXT_SIZE);
50
  });
51
  });
52
 
src/app/App.tsx CHANGED
@@ -32,6 +32,7 @@ import type {
32
  ToolRun,
33
  } from '../lib/contracts';
34
  import { formatBytes } from '../lib/format';
 
35
  import {
36
  conversationTitle,
37
  normalizeConversationModelId,
@@ -49,7 +50,7 @@ const MAX_TOOL_ROUNDS = 5;
49
  const MAX_TOOL_CALLS_PER_ROUND = 4;
50
  const MAX_TOOL_CALLS_TOTAL = 12;
51
  export const DEFAULT_MODEL_ID: ModelId = 'bonsai-27b';
52
- export const DEFAULT_CONTEXT_SIZE = 4000;
53
 
54
  const MODEL_DECOR: Record<ModelTierId, {
55
  id: ModelId;
@@ -87,7 +88,7 @@ const INITIAL_SETTINGS: RuntimeSettings = {
87
  backend: 'auto',
88
  contextSize: DEFAULT_CONTEXT_SIZE,
89
  temperature: 0.7,
90
- maxTokens: 2000,
91
  systemPrompt: 'You are Bonsai, a private browser-local assistant. Be concise and evidence-led. Use a client tool only when it materially improves the answer, and never claim a tool succeeded before its response arrives.',
92
  };
93
 
@@ -932,7 +933,7 @@ export function App() {
932
  const calls = result.toolCalls.length > 0 ? result.toolCalls : fallbackToolCalls(thought.content);
933
  if (toolPlan.requiredToolName && calls.length === 0) {
934
  outputError = (result.toolCallError || result.finishReason === 'length')
935
- ? `The ${toolPlan.requiredToolName} call ended before its arguments were complete. Regenerate, shorten the request, or clear earlier turns so the 2,000-token completion budget fits the context.`
936
  : `The model did not call the required ${toolPlan.requiredToolName} tool for this artifact request.`;
937
  }
938
  if (toolPlan.requiredToolName && calls.some((call) => call.function.name !== toolPlan.requiredToolName)) {
 
32
  ToolRun,
33
  } from '../lib/contracts';
34
  import { formatBytes } from '../lib/format';
35
+ import { DEFAULT_CONTEXT_SIZE, DEFAULT_MAX_TOKENS } from '../lib/runtime-defaults';
36
  import {
37
  conversationTitle,
38
  normalizeConversationModelId,
 
50
  const MAX_TOOL_CALLS_PER_ROUND = 4;
51
  const MAX_TOOL_CALLS_TOTAL = 12;
52
  export const DEFAULT_MODEL_ID: ModelId = 'bonsai-27b';
53
+ export { DEFAULT_CONTEXT_SIZE, DEFAULT_MAX_TOKENS } from '../lib/runtime-defaults';
54
 
55
  const MODEL_DECOR: Record<ModelTierId, {
56
  id: ModelId;
 
88
  backend: 'auto',
89
  contextSize: DEFAULT_CONTEXT_SIZE,
90
  temperature: 0.7,
91
+ maxTokens: DEFAULT_MAX_TOKENS,
92
  systemPrompt: 'You are Bonsai, a private browser-local assistant. Be concise and evidence-led. Use a client tool only when it materially improves the answer, and never claim a tool succeeded before its response arrives.',
93
  };
94
 
 
933
  const calls = result.toolCalls.length > 0 ? result.toolCalls : fallbackToolCalls(thought.content);
934
  if (toolPlan.requiredToolName && calls.length === 0) {
935
  outputError = (result.toolCallError || result.finishReason === 'length')
936
+ ? `The ${toolPlan.requiredToolName} call ended before its arguments were complete. Regenerate, shorten the request, or clear earlier turns so the ${DEFAULT_MAX_TOKENS.toLocaleString()}-token completion ceiling fits the context.`
937
  : `The model did not call the required ${toolPlan.requiredToolName} tool for this artifact request.`;
938
  }
939
  if (toolPlan.requiredToolName && calls.some((call) => call.function.name !== toolPlan.requiredToolName)) {
src/app/components/SettingsSheet.tsx CHANGED
@@ -1,5 +1,6 @@
1
  import { useEffect, useRef } from 'react';
2
  import type { ModelOption, RuntimeSettings, TelemetrySnapshot } from '../../lib/contracts';
 
3
  import type { ThemeMode } from '../theme';
4
 
5
  interface SettingsSheetProps {
@@ -103,8 +104,8 @@ export function SettingsSheet({
103
 
104
  <label className="range-setting">
105
  <span><small>Maximum completion</small><strong>{settings.maxTokens} tokens</strong></span>
106
- <input type="range" min="64" max="2000" step="16" value={settings.maxTokens} onChange={(event) => update('maxTokens', Number(event.target.value))} />
107
- <span className="range-limits"><i>64</i><i>2,000</i></span>
108
  </label>
109
 
110
  <label className="system-prompt-setting">
 
1
  import { useEffect, useRef } from 'react';
2
  import type { ModelOption, RuntimeSettings, TelemetrySnapshot } from '../../lib/contracts';
3
+ import { DEFAULT_MAX_TOKENS } from '../../lib/runtime-defaults';
4
  import type { ThemeMode } from '../theme';
5
 
6
  interface SettingsSheetProps {
 
104
 
105
  <label className="range-setting">
106
  <span><small>Maximum completion</small><strong>{settings.maxTokens} tokens</strong></span>
107
+ <input type="range" min="64" max={DEFAULT_MAX_TOKENS} step="16" value={settings.maxTokens} onChange={(event) => update('maxTokens', Number(event.target.value))} />
108
+ <span className="range-limits"><i>64</i><i>{DEFAULT_MAX_TOKENS.toLocaleString()}</i></span>
109
  </label>
110
 
111
  <label className="system-prompt-setting">
src/engine/device-gate.test.ts CHANGED
@@ -193,13 +193,13 @@ describe('evaluateModelContextPolicy', () => {
193
  ['f16 KV', { ...longContextExperiment, kvCacheType: 'f16' as const }],
194
  ['auto backend', { ...longContextExperiment, requestedBackend: 'auto' as const }],
195
  ['release defaults', { ...longContextExperiment, tuningScope: 'release-defaults' as const }],
196
- ])('keeps 27B at the 4,000-token release limit for %s', (_label, input) => {
197
  expect(evaluateModelContextPolicy(model27b, 8_448, input)).toEqual({
198
  allowed: false,
199
- limit: 4_000,
200
  });
201
- expect(evaluateModelContextPolicy(model27b, 4_000, input).allowed).toBe(true);
202
- expect(evaluateModelContextPolicy(model27b, 4_001, input).allowed).toBe(false);
203
  });
204
 
205
  it('leaves dense-tier manifest limits unchanged', () => {
 
193
  ['f16 KV', { ...longContextExperiment, kvCacheType: 'f16' as const }],
194
  ['auto backend', { ...longContextExperiment, requestedBackend: 'auto' as const }],
195
  ['release defaults', { ...longContextExperiment, tuningScope: 'release-defaults' as const }],
196
+ ])('keeps 27B at the 4,096-token release limit for %s', (_label, input) => {
197
  expect(evaluateModelContextPolicy(model27b, 8_448, input)).toEqual({
198
  allowed: false,
199
+ limit: 4_096,
200
  });
201
+ expect(evaluateModelContextPolicy(model27b, 4_096, input).allowed).toBe(true);
202
+ expect(evaluateModelContextPolicy(model27b, 4_097, input).allowed).toBe(false);
203
  });
204
 
205
  it('leaves dense-tier manifest limits unchanged', () => {
src/engine/device-gate.ts CHANGED
@@ -1,4 +1,5 @@
1
  import type { ManifestModelV2 } from './manifest';
 
2
  import type {
3
  BenchmarkFlashMode,
4
  BenchmarkKvCacheType,
@@ -10,7 +11,7 @@ import type {
10
  } from './protocol';
11
  import type { BackendReport } from './native-log';
12
 
13
- export const MODEL_27B_RELEASE_CONTEXT_LIMIT = 4_000;
14
  export const MODEL_27B_BENCHMARK_CONTEXT_LIMIT = 8_448;
15
 
16
  export interface ModelContextPolicyInput {
 
1
  import type { ManifestModelV2 } from './manifest';
2
+ import { DEFAULT_CONTEXT_SIZE } from '../lib/runtime-defaults';
3
  import type {
4
  BenchmarkFlashMode,
5
  BenchmarkKvCacheType,
 
11
  } from './protocol';
12
  import type { BackendReport } from './native-log';
13
 
14
+ export const MODEL_27B_RELEASE_CONTEXT_LIMIT = DEFAULT_CONTEXT_SIZE;
15
  export const MODEL_27B_BENCHMARK_CONTEXT_LIMIT = 8_448;
16
 
17
  export interface ModelContextPolicyInput {
src/engine/runtime.ts CHANGED
@@ -8,6 +8,7 @@ import {
8
  type ResultTimings,
9
  } from '../../vendor/wllama-bonsai/esm/index.js';
10
  import runtimeSource from '../../vendor/wllama-bonsai/SOURCE.json';
 
11
  import {
12
  assertBackendPolicy,
13
  estimateStorage,
@@ -1357,7 +1358,7 @@ export class BrowserEngineRuntime {
1357
  try {
1358
  const completionOptions = {
1359
  messages: params.messages as ChatCompletionMessage[],
1360
- max_tokens: params.maxTokens ?? 2_000,
1361
  temperature: params.temperature ?? 0,
1362
  top_p: params.topP,
1363
  top_k: params.topK ?? 1,
 
8
  type ResultTimings,
9
  } from '../../vendor/wllama-bonsai/esm/index.js';
10
  import runtimeSource from '../../vendor/wllama-bonsai/SOURCE.json';
11
+ import { DEFAULT_MAX_TOKENS } from '../lib/runtime-defaults';
12
  import {
13
  assertBackendPolicy,
14
  estimateStorage,
 
1358
  try {
1359
  const completionOptions = {
1360
  messages: params.messages as ChatCompletionMessage[],
1361
+ max_tokens: params.maxTokens ?? DEFAULT_MAX_TOKENS,
1362
  temperature: params.temperature ?? 0,
1363
  top_p: params.topP,
1364
  top_k: params.topK ?? 1,
src/lib/runtime-defaults.ts ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ export const DEFAULT_CONTEXT_SIZE = 4_096;
2
+ export const DEFAULT_MAX_TOKENS = 4_096;
src/lib/tool-routing.test.ts CHANGED
@@ -29,7 +29,7 @@ describe('HTML artifact intent routing', () => {
29
  expect(plan.tools?.map((tool) => tool.function.name)).toEqual(['html_artifact']);
30
  expect(plan.requiredToolName).toBe('html_artifact');
31
  expect(plan.minimumMaxTokens).toBe(HTML_ARTIFACT_MIN_TOKENS);
32
- expect(plan.minimumMaxTokens).toBe(2_000);
33
  expect(plan.systemInstruction).toContain('compact enough to finish');
34
  expect(plan.systemInstruction).toContain('Do not answer with a Markdown code block');
35
  });
 
29
  expect(plan.tools?.map((tool) => tool.function.name)).toEqual(['html_artifact']);
30
  expect(plan.requiredToolName).toBe('html_artifact');
31
  expect(plan.minimumMaxTokens).toBe(HTML_ARTIFACT_MIN_TOKENS);
32
+ expect(plan.minimumMaxTokens).toBe(4_096);
33
  expect(plan.systemInstruction).toContain('compact enough to finish');
34
  expect(plan.systemInstruction).toContain('Do not answer with a Markdown code block');
35
  });
src/lib/tool-routing.ts CHANGED
@@ -1,7 +1,8 @@
1
  import type { EngineChatTool } from '../engine';
2
  import { AGENT_TOOLS, HTML_ARTIFACT_TOOL } from './agent-tools';
 
3
 
4
- export const HTML_ARTIFACT_MIN_TOKENS = 2000;
5
 
6
  const CREATION_VERB = /(?:\b(?:build|code|create|design|develop|generate|implement|make|write)\b|(?:созда(?:й|йте)|сдела(?:й|йте)|напиш(?:и|ите)|сгенерир(?:уй|уйте)|сверста(?:й|йте)|реализ(?:уй|уйте)|разработа(?:й|йте)|собер(?:и|ите)|постро(?:й|йте)))/iu;
7
  const HTML_PRODUCT_TARGET = /(?:\bweb[\s-]?(?:app|application|page|site)\b|\bwebsite\b|\blanding[\s-]?page\b|(?:веб|html)[\s-]?(?:приложени\w*|страниц\w*|сайт\w*|интерфейс\w*)|(?:^|[\s,.:;!?])(?:сайт\w*|лендинг\w*))/iu;
 
1
  import type { EngineChatTool } from '../engine';
2
  import { AGENT_TOOLS, HTML_ARTIFACT_TOOL } from './agent-tools';
3
+ import { DEFAULT_MAX_TOKENS } from './runtime-defaults';
4
 
5
+ export const HTML_ARTIFACT_MIN_TOKENS = DEFAULT_MAX_TOKENS;
6
 
7
  const CREATION_VERB = /(?:\b(?:build|code|create|design|develop|generate|implement|make|write)\b|(?:созда(?:й|йте)|сдела(?:й|йте)|напиш(?:и|ите)|сгенерир(?:уй|уйте)|сверста(?:й|йте)|реализ(?:уй|уйте)|разработа(?:й|йте)|собер(?:и|ите)|постро(?:й|йте)))/iu;
8
  const HTML_PRODUCT_TARGET = /(?:\bweb[\s-]?(?:app|application|page|site)\b|\bwebsite\b|\blanding[\s-]?page\b|(?:веб|html)[\s-]?(?:приложени\w*|страниц\w*|сайт\w*|интерфейс\w*)|(?:^|[\s,.:;!?])(?:сайт\w*|лендинг\w*))/iu;