/** * Offline tests for the prompted protocol. No network, no inference credits. * * Every input string below is either a real completion captured from a small * model, or a malformation of the kind they demonstrably produce. * * Run: node --test tests/ */ import { test } from "node:test"; import assert from "node:assert/strict"; import { parseToolCalls, splitReasoning, coerceArgs, foldSystemIntoUser, mergeAlternating, buildSystemPrompt, formatToolResult, } from "../protocol.js"; import { TOOL_SCHEMAS } from "../tools.js"; const S = TOOL_SCHEMAS; /* ---------------------------------------------------------------- * * reasoning splitting * ---------------------------------------------------------------- */ test("splitReasoning handles a paired think block", () => { const { reasoning, text } = splitReasoning("weighing it upHello."); assert.equal(reasoning, "weighing it up"); assert.equal(text, "Hello."); }); test("splitReasoning handles an ORPHAN closing tag (real Qwen3.5 behaviour)", () => { // Captured shape: reasoning, then , with no opening tag at all. const raw = "The user wants a price. I should call get_price.\n\n\nBitcoin is $64,779."; const { reasoning, text } = splitReasoning(raw); assert.match(reasoning, /I should call get_price/); assert.equal(text, "Bitcoin is $64,779."); assert.ok(!text.includes(""), "closing tag must not leak into visible text"); }); test("splitReasoning handles an unterminated open tag (truncated output)", () => { const { reasoning, text } = splitReasoning("Answer soon.still thinking when cut off"); assert.equal(text, "Answer soon."); assert.match(reasoning, /still thinking/); }); /* ---------------------------------------------------------------- * * parsing: the happy path and the many unhappy ones * ---------------------------------------------------------------- */ test("parses a well-formed tagged call", () => { const raw = 'I need the price.\n{"name": "get_price", "arguments": {"coin_id": "bitcoin"}}'; const { calls } = parseToolCalls(raw, S); assert.equal(calls.length, 1); assert.deepEqual(calls[0], { name: "get_price", arguments: { coin_id: "bitcoin" } }); }); test("parses an UNTERMINATED call (stop sequence ate the closing tag)", () => { // This is the normal case in production: stop=[""]. const raw = '{"name": "get_price", "arguments": {"coin_id": "solana"}}'; const { calls } = parseToolCalls(raw, S); assert.equal(calls.length, 1); assert.equal(calls[0].arguments.coin_id, "solana"); }); test("parses two parallel calls in one completion", () => { const raw = 'Need both.\n{"name":"get_price","arguments":{"coin_id":"ethereum"}}\n' + '{"name":"get_price","arguments":{"coin_id":"solana"}}'; const { calls } = parseToolCalls(raw, S); assert.equal(calls.length, 2); assert.deepEqual(calls.map((c) => c.arguments.coin_id), ["ethereum", "solana"]); }); test("parses a ```json fenced call", () => { const raw = 'Let me look.\n```json\n{"name": "search_coin", "arguments": {"query": "doge"}}\n```'; const { calls } = parseToolCalls(raw, S); assert.equal(calls.length, 1); assert.equal(calls[0].name, "search_coin"); }); test("parses a bare object with no tags or fences", () => { const raw = 'I will call {"name": "get_coin_info", "arguments": {"coin_id": "cardano"}} now.'; const { calls } = parseToolCalls(raw, S); assert.equal(calls.length, 1); assert.equal(calls[0].name, "get_coin_info"); }); test("repairs single quotes, trailing commas and Python literals", () => { const raw = "{'name': 'get_price', 'arguments': {'coin_id': 'bitcoin',},}"; const { calls } = parseToolCalls(raw, S); assert.equal(calls.length, 1); assert.deepEqual(calls[0].arguments, { coin_id: "bitcoin" }); }); test("repairs unquoted keys", () => { const raw = '{name: "get_price", arguments: {coin_id: "bitcoin"}}'; const { calls } = parseToolCalls(raw, S); assert.equal(calls.length, 1); assert.equal(calls[0].arguments.coin_id, "bitcoin"); }); test("accepts alias key names (tool / parameters)", () => { const raw = '{"tool": "get_price", "parameters": {"coin_id": "bitcoin"}}'; const { calls } = parseToolCalls(raw, S); assert.equal(calls[0].name, "get_price"); assert.equal(calls[0].arguments.coin_id, "bitcoin"); }); test("accepts arguments delivered as a JSON string", () => { const raw = '{"name":"get_price","arguments":"{\\"coin_id\\":\\"bitcoin\\"}"}'; const { calls } = parseToolCalls(raw, S); assert.equal(calls[0].arguments.coin_id, "bitcoin"); }); test("unwraps a nested function/tool_call envelope", () => { const raw = '{"function": {"name": "get_price", "arguments": {"coin_id": "bitcoin"}}}'; const { calls } = parseToolCalls(raw, S); assert.equal(calls.length, 1); assert.equal(calls[0].name, "get_price"); }); test("deduplicates a call that is both fenced AND tagged", () => { const raw = '{"name":"get_price","arguments":{"coin_id":"bitcoin"}}\n' + '```json\n{"name":"get_price","arguments":{"coin_id":"bitcoin"}}\n```'; const { calls } = parseToolCalls(raw, S); assert.equal(calls.length, 1, "must not fire the same call twice"); }); test("plain prose yields no calls and is returned as visible text", () => { const { calls, text } = parseToolCalls("Bitcoin is trading at $64,779 right now.", S); assert.equal(calls.length, 0); assert.match(text, /64,779/); }); test("the observed Qwen3.5-2B failure mode yields no calls, not a crash", () => { // Real captured output: it mimicked the few-shot's prose line and stopped. const { calls, text } = parseToolCalls("I need the live price for AAPL.\n", S); assert.equal(calls.length, 0); assert.match(text, /I need the live price/); }); test("malformed beyond repair does not throw", () => { const { calls } = parseToolCalls('{"name": "get_price", "argum', S); assert.ok(Array.isArray(calls)); }); test("reasoning plus a call in one completion: both are recovered", () => { const raw = 'The user asks for a price, so I need get_price.\n\n\nFetching that now.\n' + '{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'; const { reasoning, calls, text } = parseToolCalls(raw, S); assert.match(reasoning, /so I need get_price/); assert.equal(calls.length, 1); assert.ok(!text.includes("tool_call"), "call block must be stripped from visible prose"); }); /* ---------------------------------------------------------------- * * argument coercion * ---------------------------------------------------------------- */ test("coerces stringified numbers and integers per schema", () => { const out = coerceArgs("convert_currency", { amount: "1919.55", from_currency: "USD", to_currency: "EUR" }, S); assert.equal(out.amount, 1919.55); assert.equal(typeof out.amount, "number"); const chart = coerceArgs("get_market_chart", { coin_id: "bitcoin", days: "30" }, S); assert.equal(chart.days, 30); assert.equal(typeof chart.days, "number"); }); test("coerces a numeric value into a string field", () => { const out = coerceArgs("search_coin", { query: 123 }, S); assert.equal(out.query, "123"); }); test("coercion leaves unknown tools and unknown keys untouched", () => { assert.deepEqual(coerceArgs("no_such_tool", { a: "1" }, S), { a: "1" }); assert.deepEqual(coerceArgs("get_price", { mystery: "x" }, S), { mystery: "x" }); }); /* ---------------------------------------------------------------- * * template shims * ---------------------------------------------------------------- */ test("foldSystemIntoUser merges the system prompt into the first user turn", () => { const out = foldSystemIntoUser([ { role: "system", content: "SYS" }, { role: "user", content: "hi" }, ]); assert.equal(out.length, 1); assert.equal(out[0].role, "user"); assert.match(out[0].content, /SYS/); assert.match(out[0].content, /hi/); assert.ok(!out.some((m) => m.role === "system"), "no system role may survive"); }); test("mergeAlternating collapses consecutive same-role turns", () => { const out = mergeAlternating([ { role: "user", content: "a" }, { role: "user", content: "b" }, { role: "assistant", content: "c" }, ]); assert.equal(out.length, 2); assert.equal(out[0].content, "a\nb"); // gemma-2 requires strict alternation; verify that is what we produced. for (let i = 1; i < out.length; i++) { assert.notEqual(out[i].role, out[i - 1].role); } }); /* ---------------------------------------------------------------- * * prompt assembly * ---------------------------------------------------------------- */ test("system prompt embeds every tool name and is valid-schema-bearing", () => { const p = buildSystemPrompt(S); for (const s of S) assert.ok(p.includes(s.function.name), `missing ${s.function.name}`); assert.ok(p.includes("")); assert.ok(!/[Ѐ-ӿ]/.test(p), "prompt must not contain stray Cyrillic"); }); test("formatToolResult round-trips through the parser as non-call text", () => { const line = formatToolResult("get_price", { coin_id: "bitcoin", price: 1 }); assert.match(line, /^TOOL_RESULT get_price /); // A result echoed back must never be mistaken for a new tool call. const { calls } = parseToolCalls(line, S); assert.equal(calls.length, 0, "a TOOL_RESULT line must not parse as a call"); });