Spaces:
Running
Running
Valeriy Selitskiy commited on
Commit ·
36a170e
1
Parent(s): 432e8f2
Use 4096-token release defaults
Browse files- public/manifest/models.json +1 -1
- src/app/App.test.ts +8 -2
- src/app/App.tsx +4 -3
- src/app/components/SettingsSheet.tsx +3 -2
- src/engine/device-gate.test.ts +4 -4
- src/engine/device-gate.ts +2 -1
- src/engine/runtime.ts +2 -1
- src/lib/runtime-defaults.ts +2 -0
- src/lib/tool-routing.test.ts +1 -1
- src/lib/tool-routing.ts +2 -1
public/manifest/models.json
CHANGED
|
@@ -218,7 +218,7 @@
|
|
| 218 |
],
|
| 219 |
"downloadBytes": 3803453440,
|
| 220 |
"contextLength": 262144,
|
| 221 |
-
"defaultContext":
|
| 222 |
"cpuFallback": false,
|
| 223 |
"largestTensorBytes": 178790400,
|
| 224 |
"requiredLimits": {
|
|
|
|
| 218 |
],
|
| 219 |
"downloadBytes": 3803453440,
|
| 220 |
"contextLength": 262144,
|
| 221 |
+
"defaultContext": 4096,
|
| 222 |
"cpuFallback": false,
|
| 223 |
"largestTensorBytes": 178790400,
|
| 224 |
"requiredLimits": {
|
src/app/App.test.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
| 1 |
import { describe, expect, it } from 'vitest';
|
|
|
|
| 2 |
import { EngineClientError } from '../engine';
|
| 3 |
import {
|
| 4 |
DEFAULT_CONTEXT_SIZE,
|
|
|
|
| 5 |
DEFAULT_MODEL_ID,
|
| 6 |
modelArchitectureLabel,
|
| 7 |
publishBackendReport,
|
|
@@ -39,8 +41,12 @@ describe('App release defaults', () => {
|
|
| 39 |
expect(modelArchitectureLabel('qwen35')).toBe('Qwen3.5 hybrid');
|
| 40 |
});
|
| 41 |
|
| 42 |
-
it('
|
| 43 |
-
expect(DEFAULT_CONTEXT_SIZE).toBe(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
});
|
| 45 |
});
|
| 46 |
|
|
|
|
| 1 |
import { describe, expect, it } from 'vitest';
|
| 2 |
+
import releaseManifest from '../../public/manifest/models.json';
|
| 3 |
import { EngineClientError } from '../engine';
|
| 4 |
import {
|
| 5 |
DEFAULT_CONTEXT_SIZE,
|
| 6 |
+
DEFAULT_MAX_TOKENS,
|
| 7 |
DEFAULT_MODEL_ID,
|
| 8 |
modelArchitectureLabel,
|
| 9 |
publishBackendReport,
|
|
|
|
| 41 |
expect(modelArchitectureLabel('qwen35')).toBe('Qwen3.5 hybrid');
|
| 42 |
});
|
| 43 |
|
| 44 |
+
it('uses one 4K ceiling for context allocation and completion requests', () => {
|
| 45 |
+
expect(DEFAULT_CONTEXT_SIZE).toBe(4096);
|
| 46 |
+
expect(DEFAULT_MAX_TOKENS).toBe(4096);
|
| 47 |
+
expect(DEFAULT_CONTEXT_SIZE).toBe(DEFAULT_MAX_TOKENS);
|
| 48 |
+
expect(releaseManifest.models.find((model) => model.id === '27b')?.defaultContext)
|
| 49 |
+
.toBe(DEFAULT_CONTEXT_SIZE);
|
| 50 |
});
|
| 51 |
});
|
| 52 |
|
src/app/App.tsx
CHANGED
|
@@ -32,6 +32,7 @@ import type {
|
|
| 32 |
ToolRun,
|
| 33 |
} from '../lib/contracts';
|
| 34 |
import { formatBytes } from '../lib/format';
|
|
|
|
| 35 |
import {
|
| 36 |
conversationTitle,
|
| 37 |
normalizeConversationModelId,
|
|
@@ -49,7 +50,7 @@ const MAX_TOOL_ROUNDS = 5;
|
|
| 49 |
const MAX_TOOL_CALLS_PER_ROUND = 4;
|
| 50 |
const MAX_TOOL_CALLS_TOTAL = 12;
|
| 51 |
export const DEFAULT_MODEL_ID: ModelId = 'bonsai-27b';
|
| 52 |
-
export
|
| 53 |
|
| 54 |
const MODEL_DECOR: Record<ModelTierId, {
|
| 55 |
id: ModelId;
|
|
@@ -87,7 +88,7 @@ const INITIAL_SETTINGS: RuntimeSettings = {
|
|
| 87 |
backend: 'auto',
|
| 88 |
contextSize: DEFAULT_CONTEXT_SIZE,
|
| 89 |
temperature: 0.7,
|
| 90 |
-
maxTokens:
|
| 91 |
systemPrompt: 'You are Bonsai, a private browser-local assistant. Be concise and evidence-led. Use a client tool only when it materially improves the answer, and never claim a tool succeeded before its response arrives.',
|
| 92 |
};
|
| 93 |
|
|
@@ -932,7 +933,7 @@ export function App() {
|
|
| 932 |
const calls = result.toolCalls.length > 0 ? result.toolCalls : fallbackToolCalls(thought.content);
|
| 933 |
if (toolPlan.requiredToolName && calls.length === 0) {
|
| 934 |
outputError = (result.toolCallError || result.finishReason === 'length')
|
| 935 |
-
? `The ${toolPlan.requiredToolName} call ended before its arguments were complete. Regenerate, shorten the request, or clear earlier turns so the
|
| 936 |
: `The model did not call the required ${toolPlan.requiredToolName} tool for this artifact request.`;
|
| 937 |
}
|
| 938 |
if (toolPlan.requiredToolName && calls.some((call) => call.function.name !== toolPlan.requiredToolName)) {
|
|
|
|
| 32 |
ToolRun,
|
| 33 |
} from '../lib/contracts';
|
| 34 |
import { formatBytes } from '../lib/format';
|
| 35 |
+
import { DEFAULT_CONTEXT_SIZE, DEFAULT_MAX_TOKENS } from '../lib/runtime-defaults';
|
| 36 |
import {
|
| 37 |
conversationTitle,
|
| 38 |
normalizeConversationModelId,
|
|
|
|
| 50 |
const MAX_TOOL_CALLS_PER_ROUND = 4;
|
| 51 |
const MAX_TOOL_CALLS_TOTAL = 12;
|
| 52 |
export const DEFAULT_MODEL_ID: ModelId = 'bonsai-27b';
|
| 53 |
+
export { DEFAULT_CONTEXT_SIZE, DEFAULT_MAX_TOKENS } from '../lib/runtime-defaults';
|
| 54 |
|
| 55 |
const MODEL_DECOR: Record<ModelTierId, {
|
| 56 |
id: ModelId;
|
|
|
|
| 88 |
backend: 'auto',
|
| 89 |
contextSize: DEFAULT_CONTEXT_SIZE,
|
| 90 |
temperature: 0.7,
|
| 91 |
+
maxTokens: DEFAULT_MAX_TOKENS,
|
| 92 |
systemPrompt: 'You are Bonsai, a private browser-local assistant. Be concise and evidence-led. Use a client tool only when it materially improves the answer, and never claim a tool succeeded before its response arrives.',
|
| 93 |
};
|
| 94 |
|
|
|
|
| 933 |
const calls = result.toolCalls.length > 0 ? result.toolCalls : fallbackToolCalls(thought.content);
|
| 934 |
if (toolPlan.requiredToolName && calls.length === 0) {
|
| 935 |
outputError = (result.toolCallError || result.finishReason === 'length')
|
| 936 |
+
? `The ${toolPlan.requiredToolName} call ended before its arguments were complete. Regenerate, shorten the request, or clear earlier turns so the ${DEFAULT_MAX_TOKENS.toLocaleString()}-token completion ceiling fits the context.`
|
| 937 |
: `The model did not call the required ${toolPlan.requiredToolName} tool for this artifact request.`;
|
| 938 |
}
|
| 939 |
if (toolPlan.requiredToolName && calls.some((call) => call.function.name !== toolPlan.requiredToolName)) {
|
src/app/components/SettingsSheet.tsx
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
import { useEffect, useRef } from 'react';
|
| 2 |
import type { ModelOption, RuntimeSettings, TelemetrySnapshot } from '../../lib/contracts';
|
|
|
|
| 3 |
import type { ThemeMode } from '../theme';
|
| 4 |
|
| 5 |
interface SettingsSheetProps {
|
|
@@ -103,8 +104,8 @@ export function SettingsSheet({
|
|
| 103 |
|
| 104 |
<label className="range-setting">
|
| 105 |
<span><small>Maximum completion</small><strong>{settings.maxTokens} tokens</strong></span>
|
| 106 |
-
<input type="range" min="64" max=
|
| 107 |
-
<span className="range-limits"><i>64</i><i>
|
| 108 |
</label>
|
| 109 |
|
| 110 |
<label className="system-prompt-setting">
|
|
|
|
| 1 |
import { useEffect, useRef } from 'react';
|
| 2 |
import type { ModelOption, RuntimeSettings, TelemetrySnapshot } from '../../lib/contracts';
|
| 3 |
+
import { DEFAULT_MAX_TOKENS } from '../../lib/runtime-defaults';
|
| 4 |
import type { ThemeMode } from '../theme';
|
| 5 |
|
| 6 |
interface SettingsSheetProps {
|
|
|
|
| 104 |
|
| 105 |
<label className="range-setting">
|
| 106 |
<span><small>Maximum completion</small><strong>{settings.maxTokens} tokens</strong></span>
|
| 107 |
+
<input type="range" min="64" max={DEFAULT_MAX_TOKENS} step="16" value={settings.maxTokens} onChange={(event) => update('maxTokens', Number(event.target.value))} />
|
| 108 |
+
<span className="range-limits"><i>64</i><i>{DEFAULT_MAX_TOKENS.toLocaleString()}</i></span>
|
| 109 |
</label>
|
| 110 |
|
| 111 |
<label className="system-prompt-setting">
|
src/engine/device-gate.test.ts
CHANGED
|
@@ -193,13 +193,13 @@ describe('evaluateModelContextPolicy', () => {
|
|
| 193 |
['f16 KV', { ...longContextExperiment, kvCacheType: 'f16' as const }],
|
| 194 |
['auto backend', { ...longContextExperiment, requestedBackend: 'auto' as const }],
|
| 195 |
['release defaults', { ...longContextExperiment, tuningScope: 'release-defaults' as const }],
|
| 196 |
-
])('keeps 27B at the 4,
|
| 197 |
expect(evaluateModelContextPolicy(model27b, 8_448, input)).toEqual({
|
| 198 |
allowed: false,
|
| 199 |
-
limit:
|
| 200 |
});
|
| 201 |
-
expect(evaluateModelContextPolicy(model27b,
|
| 202 |
-
expect(evaluateModelContextPolicy(model27b,
|
| 203 |
});
|
| 204 |
|
| 205 |
it('leaves dense-tier manifest limits unchanged', () => {
|
|
|
|
| 193 |
['f16 KV', { ...longContextExperiment, kvCacheType: 'f16' as const }],
|
| 194 |
['auto backend', { ...longContextExperiment, requestedBackend: 'auto' as const }],
|
| 195 |
['release defaults', { ...longContextExperiment, tuningScope: 'release-defaults' as const }],
|
| 196 |
+
])('keeps 27B at the 4,096-token release limit for %s', (_label, input) => {
|
| 197 |
expect(evaluateModelContextPolicy(model27b, 8_448, input)).toEqual({
|
| 198 |
allowed: false,
|
| 199 |
+
limit: 4_096,
|
| 200 |
});
|
| 201 |
+
expect(evaluateModelContextPolicy(model27b, 4_096, input).allowed).toBe(true);
|
| 202 |
+
expect(evaluateModelContextPolicy(model27b, 4_097, input).allowed).toBe(false);
|
| 203 |
});
|
| 204 |
|
| 205 |
it('leaves dense-tier manifest limits unchanged', () => {
|
src/engine/device-gate.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
import type { ManifestModelV2 } from './manifest';
|
|
|
|
| 2 |
import type {
|
| 3 |
BenchmarkFlashMode,
|
| 4 |
BenchmarkKvCacheType,
|
|
@@ -10,7 +11,7 @@ import type {
|
|
| 10 |
} from './protocol';
|
| 11 |
import type { BackendReport } from './native-log';
|
| 12 |
|
| 13 |
-
export const MODEL_27B_RELEASE_CONTEXT_LIMIT =
|
| 14 |
export const MODEL_27B_BENCHMARK_CONTEXT_LIMIT = 8_448;
|
| 15 |
|
| 16 |
export interface ModelContextPolicyInput {
|
|
|
|
| 1 |
import type { ManifestModelV2 } from './manifest';
|
| 2 |
+
import { DEFAULT_CONTEXT_SIZE } from '../lib/runtime-defaults';
|
| 3 |
import type {
|
| 4 |
BenchmarkFlashMode,
|
| 5 |
BenchmarkKvCacheType,
|
|
|
|
| 11 |
} from './protocol';
|
| 12 |
import type { BackendReport } from './native-log';
|
| 13 |
|
| 14 |
+
export const MODEL_27B_RELEASE_CONTEXT_LIMIT = DEFAULT_CONTEXT_SIZE;
|
| 15 |
export const MODEL_27B_BENCHMARK_CONTEXT_LIMIT = 8_448;
|
| 16 |
|
| 17 |
export interface ModelContextPolicyInput {
|
src/engine/runtime.ts
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
| 8 |
type ResultTimings,
|
| 9 |
} from '../../vendor/wllama-bonsai/esm/index.js';
|
| 10 |
import runtimeSource from '../../vendor/wllama-bonsai/SOURCE.json';
|
|
|
|
| 11 |
import {
|
| 12 |
assertBackendPolicy,
|
| 13 |
estimateStorage,
|
|
@@ -1357,7 +1358,7 @@ export class BrowserEngineRuntime {
|
|
| 1357 |
try {
|
| 1358 |
const completionOptions = {
|
| 1359 |
messages: params.messages as ChatCompletionMessage[],
|
| 1360 |
-
max_tokens: params.maxTokens ??
|
| 1361 |
temperature: params.temperature ?? 0,
|
| 1362 |
top_p: params.topP,
|
| 1363 |
top_k: params.topK ?? 1,
|
|
|
|
| 8 |
type ResultTimings,
|
| 9 |
} from '../../vendor/wllama-bonsai/esm/index.js';
|
| 10 |
import runtimeSource from '../../vendor/wllama-bonsai/SOURCE.json';
|
| 11 |
+
import { DEFAULT_MAX_TOKENS } from '../lib/runtime-defaults';
|
| 12 |
import {
|
| 13 |
assertBackendPolicy,
|
| 14 |
estimateStorage,
|
|
|
|
| 1358 |
try {
|
| 1359 |
const completionOptions = {
|
| 1360 |
messages: params.messages as ChatCompletionMessage[],
|
| 1361 |
+
max_tokens: params.maxTokens ?? DEFAULT_MAX_TOKENS,
|
| 1362 |
temperature: params.temperature ?? 0,
|
| 1363 |
top_p: params.topP,
|
| 1364 |
top_k: params.topK ?? 1,
|
src/lib/runtime-defaults.ts
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
export const DEFAULT_CONTEXT_SIZE = 4_096;
|
| 2 |
+
export const DEFAULT_MAX_TOKENS = 4_096;
|
src/lib/tool-routing.test.ts
CHANGED
|
@@ -29,7 +29,7 @@ describe('HTML artifact intent routing', () => {
|
|
| 29 |
expect(plan.tools?.map((tool) => tool.function.name)).toEqual(['html_artifact']);
|
| 30 |
expect(plan.requiredToolName).toBe('html_artifact');
|
| 31 |
expect(plan.minimumMaxTokens).toBe(HTML_ARTIFACT_MIN_TOKENS);
|
| 32 |
-
expect(plan.minimumMaxTokens).toBe(
|
| 33 |
expect(plan.systemInstruction).toContain('compact enough to finish');
|
| 34 |
expect(plan.systemInstruction).toContain('Do not answer with a Markdown code block');
|
| 35 |
});
|
|
|
|
| 29 |
expect(plan.tools?.map((tool) => tool.function.name)).toEqual(['html_artifact']);
|
| 30 |
expect(plan.requiredToolName).toBe('html_artifact');
|
| 31 |
expect(plan.minimumMaxTokens).toBe(HTML_ARTIFACT_MIN_TOKENS);
|
| 32 |
+
expect(plan.minimumMaxTokens).toBe(4_096);
|
| 33 |
expect(plan.systemInstruction).toContain('compact enough to finish');
|
| 34 |
expect(plan.systemInstruction).toContain('Do not answer with a Markdown code block');
|
| 35 |
});
|
src/lib/tool-routing.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
| 1 |
import type { EngineChatTool } from '../engine';
|
| 2 |
import { AGENT_TOOLS, HTML_ARTIFACT_TOOL } from './agent-tools';
|
|
|
|
| 3 |
|
| 4 |
-
export const HTML_ARTIFACT_MIN_TOKENS =
|
| 5 |
|
| 6 |
const CREATION_VERB = /(?:\b(?:build|code|create|design|develop|generate|implement|make|write)\b|(?:созда(?:й|йте)|сдела(?:й|йте)|напиш(?:и|ите)|сгенерир(?:уй|уйте)|сверста(?:й|йте)|реализ(?:уй|уйте)|разработа(?:й|йте)|собер(?:и|ите)|постро(?:й|йте)))/iu;
|
| 7 |
const HTML_PRODUCT_TARGET = /(?:\bweb[\s-]?(?:app|application|page|site)\b|\bwebsite\b|\blanding[\s-]?page\b|(?:веб|html)[\s-]?(?:приложени\w*|страниц\w*|сайт\w*|интерфейс\w*)|(?:^|[\s,.:;!?])(?:сайт\w*|лендинг\w*))/iu;
|
|
|
|
| 1 |
import type { EngineChatTool } from '../engine';
|
| 2 |
import { AGENT_TOOLS, HTML_ARTIFACT_TOOL } from './agent-tools';
|
| 3 |
+
import { DEFAULT_MAX_TOKENS } from './runtime-defaults';
|
| 4 |
|
| 5 |
+
export const HTML_ARTIFACT_MIN_TOKENS = DEFAULT_MAX_TOKENS;
|
| 6 |
|
| 7 |
const CREATION_VERB = /(?:\b(?:build|code|create|design|develop|generate|implement|make|write)\b|(?:созда(?:й|йте)|сдела(?:й|йте)|напиш(?:и|ите)|сгенерир(?:уй|уйте)|сверста(?:й|йте)|реализ(?:уй|уйте)|разработа(?:й|йте)|собер(?:и|ите)|постро(?:й|йте)))/iu;
|
| 8 |
const HTML_PRODUCT_TARGET = /(?:\bweb[\s-]?(?:app|application|page|site)\b|\bwebsite\b|\blanding[\s-]?page\b|(?:веб|html)[\s-]?(?:приложени\w*|страниц\w*|сайт\w*|интерфейс\w*)|(?:^|[\s,.:;!?])(?:сайт\w*|лендинг\w*))/iu;
|