/** * Agent-loop tests with a fully mocked network. No credits, no live APIs. * * Both the HF router and the data APIs are intercepted, so these exercise the * real control flow: native path, prompted path, the auto fallback between * them, multi-turn chaining and error recovery. */ import { test } from "node:test"; import assert from "node:assert/strict"; import { runAgent } from "../agent.js"; const realFetch = globalThis.fetch; /** * @param {Array} routerReplies one per router call, in order. * A function receives the parsed request body; an object is returned as-is. */ function mockNetwork(routerReplies) { const calls = { router: [], data: [] }; let i = 0; globalThis.fetch = async (url, init = {}) => { const u = String(url); if (u.includes("router.huggingface.co")) { const body = JSON.parse(init.body); calls.router.push(body); const reply = routerReplies[i++]; if (reply === undefined) throw new Error(`unexpected router call #${i}`); const resolved = typeof reply === "function" ? reply(body) : reply; if (resolved.__http) { return new Response(JSON.stringify({ error: { message: resolved.message } }), { status: resolved.__http, headers: { "content-type": "application/json" }, }); } return new Response(JSON.stringify(resolved), { status: 200, headers: { "content-type": "application/json" }, }); } calls.data.push(u); if (u.includes("/simple/price")) { return json({ bitcoin: { usd: 64779, usd_24h_change: 0.79, usd_market_cap: 1.3e12 } }); } if (u.includes("/search")) { return json({ coins: [{ id: "bitcoin", symbol: "btc", name: "Bitcoin", market_cap_rank: 1 }] }); } if (u.includes("open.er-api.com")) { return json({ result: "success", rates: { TRY: 42.5 }, time_last_update_utc: "now" }); } return json({}); }; return calls; } const json = (o) => new Response(JSON.stringify(o), { status: 200, headers: { "content-type": "application/json" } }); const nativeReply = (toolCalls, content = null) => ({ choices: [ { message: { role: "assistant", content, tool_calls: toolCalls.map((c, i) => ({ id: `call_${i}`, type: "function", function: { name: c.name, arguments: JSON.stringify(c.arguments) }, })), }, }, ], usage: { prompt_tokens: 10, completion_tokens: 5 }, }); const textReply = (content) => ({ choices: [{ message: { role: "assistant", content } }], usage: { prompt_tokens: 10, completion_tokens: 5 }, }); const base = { model: "test/model", token: "hf_test", userMessage: "What is the price of bitcoin?" }; test.afterEach(() => { globalThis.fetch = realFetch; }); /* ---------------------------------------------------------------- */ test("native mode: tool call, then final answer", async () => { const net = mockNetwork([ nativeReply([{ name: "get_price", arguments: { coin_id: "bitcoin" } }]), textReply("Bitcoin is at $64,779."), ]); const events = []; const out = await runAgent({ ...base, mode: "native", onEvent: (e) => events.push(e) }); assert.equal(out.answer, "Bitcoin is at $64,779."); assert.equal(out.turns, 2); assert.equal(events.filter((e) => e.type === "tool_call").length, 1); assert.equal(events.find((e) => e.type === "tool_result").result.price, 64779); // Second request must carry the assistant tool_calls turn and a tool result. const second = net.router[1]; assert.ok(second.messages.some((m) => m.role === "assistant" && m.tool_calls)); assert.ok(second.messages.some((m) => m.role === "tool")); assert.ok(second.tools, "native mode must send the tools array"); }); test("prompted mode: parses a tagged call and feeds the result back as a USER turn", async () => { const net = mockNetwork([ textReply('I need the price.\n{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'), textReply("Bitcoin is at $64,779."), ]); const out = await runAgent({ ...base, mode: "prompted", onEvent: () => {} }); assert.equal(out.answer, "Bitcoin is at $64,779."); const first = net.router[0]; assert.ok(!first.tools, "prompted mode must NOT send a tools array"); assert.deepEqual(first.stop, [""], "stop sequence is required"); assert.match(first.messages[0].content, /AVAILABLE TOOLS/); const second = net.router[1]; const fed = second.messages[second.messages.length - 1]; assert.equal(fed.role, "user", "results must come back as a user turn, not role:tool"); assert.match(fed.content, /^TOOL_RESULT get_price /); assert.ok(!second.messages.some((m) => m.role === "tool"), "no tool role in prompted mode"); }); test("auto mode falls back to prompted when the provider rejects tools", async () => { const net = mockNetwork([ { __http: 400, message: "tools are not supported for this model" }, textReply('{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'), textReply("Bitcoin is at $64,779."), ]); const events = []; const out = await runAgent({ ...base, mode: "auto", onEvent: (e) => events.push(e) }); assert.equal(out.mode, "prompted"); assert.ok(out.fellBack); assert.ok(events.some((e) => e.type === "fallback"), "the fallback must be visible in the trace"); assert.ok(!net.router[0].stop, "the first (native) attempt sends no stop sequence"); assert.deepEqual(net.router[1].stop, [""]); assert.equal(out.answer, "Bitcoin is at $64,779."); }); test("auto mode falls back when the model silently ignores `tools` and returns prose", async () => { // The dangerous case: HTTP 200, no tool_calls, just an invented answer. const net = mockNetwork([ textReply("Bitcoin is worth about twenty thousand dollars, I think."), textReply('{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'), textReply("Bitcoin is at $64,779."), ]); const out = await runAgent({ ...base, mode: "auto", onEvent: () => {} }); assert.equal(out.mode, "prompted"); assert.equal(out.answer, "Bitcoin is at $64,779."); assert.equal(net.router.length, 3); }); test("auto does NOT fall back for a genuine non-tool question", async () => { const net = mockNetwork([textReply("Hello! How can I help with market data today?")]); const out = await runAgent({ ...base, userMessage: "hello there", mode: "auto", onEvent: () => {}, }); assert.equal(out.mode, "native"); assert.equal(net.router.length, 1, "a greeting must not trigger a fallback retry"); }); test("multi-turn chaining: price, then currency conversion", async () => { mockNetwork([ textReply('{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'), textReply('{"name":"convert_currency","arguments":{"amount":64779,"from_currency":"USD","to_currency":"TRY"}}'), textReply("Bitcoin is $64,779, about 2,753,107 TRY."), ]); const events = []; const out = await runAgent({ ...base, userMessage: "What is bitcoin in Turkish lira?", mode: "prompted", onEvent: (e) => events.push(e), }); const names = events.filter((e) => e.type === "tool_call").map((e) => e.name); assert.deepEqual(names, ["get_price", "convert_currency"]); assert.equal(out.turns, 3); const conv = events.filter((e) => e.type === "tool_result").at(-1); assert.equal(conv.result.result, 2753107.5); }); test("two parallel calls in one turn are both executed", async () => { mockNetwork([ textReply( '{"name":"get_price","arguments":{"coin_id":"bitcoin"}}\n' + '{"name":"search_coin","arguments":{"query":"btc"}}' ), textReply("Done."), ]); const events = []; await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) }); assert.equal(events.filter((e) => e.type === "tool_call").length, 2); }); test("a tool error is fed back as data so the model can recover", async () => { mockNetwork([ textReply('{"name":"no_such_tool","arguments":{}}'), textReply("Sorry, that tool does not exist."), ]); const events = []; const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) }); const res = events.find((e) => e.type === "tool_result"); assert.ok(res.failed); assert.match(res.result.error, /unknown tool/); assert.equal(out.answer, "Sorry, that tool does not exist."); }); test("gemma-2: system role is folded away and turns strictly alternate", async () => { const net = mockNetwork([textReply("Hi there.")]); await runAgent({ ...base, model: "google/gemma-2-2b-it", userMessage: "hello", mode: "prompted", onEvent: () => {}, }); const msgs = net.router[0].messages; assert.ok(!msgs.some((m) => m.role === "system"), "gemma-2 rejects a system role"); assert.match(msgs[0].content, /AVAILABLE TOOLS/, "the prompt must survive the fold"); for (let i = 1; i < msgs.length; i++) { assert.notEqual(msgs[i].role, msgs[i - 1].role, "gemma-2 needs strict alternation"); } }); test("credit exhaustion surfaces a clear message and does not retry", async () => { const net = mockNetwork([{ __http: 402, message: "Payment Required" }]); await assert.rejects( () => runAgent({ ...base, mode: "auto", onEvent: () => {} }), (e) => { assert.equal(e.kind, "credits"); assert.match(e.message, /credits exhausted/i); return true; } ); assert.equal(net.router.length, 1, "402 must not trigger a prompted retry"); }); test("a bad token surfaces an auth error, not a fallback", async () => { const net = mockNetwork([{ __http: 401, message: "Invalid credentials" }]); await assert.rejects( () => runAgent({ ...base, mode: "auto", onEvent: () => {} }), (e) => { assert.equal(e.kind, "auth"); assert.match(e.message, /Token rejected/); return true; } ); assert.equal(net.router.length, 1, "401 must not trigger a prompted retry"); }); test("provider is pinned onto the model id as model:provider", async () => { const net = mockNetwork([textReply("hi")]); await runAgent({ ...base, userMessage: "hello", provider: "featherless-ai", mode: "prompted", onEvent: () => {}, }); assert.equal(net.router[0].model, "test/model:featherless-ai"); }); test("the turn cap stops a model that loops forever", async () => { const looping = Array.from({ length: 8 }, () => textReply('{"name":"get_price","arguments":{"coin_id":"bitcoin"}}') ); mockNetwork(looping); const events = []; const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) }); assert.equal(out.turns, 6); assert.match(events.at(-1).text, /maximum number of tool-calling turns/); }); /* ---------------------------------------------------------------- * * regressions found during live testing against Qwen3.5-2B * ---------------------------------------------------------------- */ test("REGRESSION: concatenated completion+error body is reported, not swallowed", async () => { // Real featherless response: HTTP 200, a valid empty completion, then an // error object appended. JSON.parse throws on this. globalThis.fetch = async () => new Response( JSON.stringify({ choices: [{ message: { role: "assistant", content: "" } }] }) + JSON.stringify({ error: { message: "No successful response received from completion service" }, }), { status: 200, headers: { "content-type": "application/json" } } ); await assert.rejects( () => runAgent({ ...base, mode: "prompted", onEvent: () => {} }), (e) => { assert.equal(e.kind, "provider"); assert.match(e.message, /No successful response received/); return true; } ); }); test("a concatenated body WITH real content is still usable", async () => { globalThis.fetch = async () => new Response( JSON.stringify({ choices: [{ message: { role: "assistant", content: "Bitcoin is $64,779." } }] }) + JSON.stringify({ error: { message: "late warning" } }), { status: 200, headers: { "content-type": "application/json" } } ); const out = await runAgent({ ...base, mode: "prompted", onEvent: () => {} }); assert.equal(out.answer, "Bitcoin is $64,779."); }); test("REGRESSION: prose-instead-of-call is nudged, then recovers", async () => { // The exact captured Qwen3.5-2B failure: it narrates intent and stops. const net = mockNetwork([ textReply("I need the live price of Bitcoin."), textReply('{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'), textReply("Bitcoin is at $64,779."), ]); const events = []; const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) }); const nudge = events.find((e) => e.type === "nudge"); assert.ok(nudge, "the nudge must be visible in the trace"); assert.match(nudge.text, /I need the live price/); assert.equal(out.answer, "Bitcoin is at $64,779."); const retry = net.router[1]; const last = retry.messages.at(-1); assert.equal(last.role, "user"); assert.match(last.content, /ONLY the tool call block/); }); test("the nudge fires at most once, then accepts the prose as final", async () => { const net = mockNetwork([ textReply("I need the live price of Bitcoin."), textReply("I really do need it."), ]); const events = []; const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) }); assert.equal(events.filter((e) => e.type === "nudge").length, 1, "must not nudge repeatedly"); assert.equal(net.router.length, 2); assert.equal(out.answer, "I really do need it."); }); test("a non-tool question is never nudged", async () => { const net = mockNetwork([textReply("Hello, how can I help?")]); await runAgent({ ...base, userMessage: "hello", mode: "prompted", onEvent: () => {} }); assert.equal(net.router.length, 1); }); test("few-shot examples are delivered as real alternating message turns", async () => { const net = mockNetwork([textReply("hi")]); await runAgent({ ...base, userMessage: "hello", mode: "prompted", onEvent: () => {} }); const msgs = net.router[0].messages; assert.equal(msgs[0].role, "system"); // The demonstrated assistant turns must contain ONLY the call: a leading prose // line is what made Qwen3.5-2B stop before emitting one. const shots = msgs.filter((m) => m.role === "assistant" && m.content.includes("")); assert.ok(shots.length >= 2, "expected few-shot assistant turns with calls"); for (const s of shots) { assert.ok(s.content.trimStart().startsWith(""), `prose before call: ${s.content}`); } for (let i = 1; i < msgs.length; i++) { if (msgs[i].role !== "system") assert.notEqual(msgs[i].role, msgs[i - 1].role); } }); test("prompted mode sheds the few-shot and retries when the provider errors", async () => { let n = 0; const seen = []; globalThis.fetch = async (url, init = {}) => { const u = String(url); if (u.includes("router.huggingface.co")) { const body = JSON.parse(init.body); seen.push(body.messages.length); n++; if (n === 1) { // The featherless shape: 200, empty completion, trailing error object. return new Response( JSON.stringify({ choices: [{ message: { role: "assistant", content: "" } }] }) + JSON.stringify({ error: { message: "No successful response received" } }), { status: 200, headers: { "content-type": "application/json" } } ); } if (n === 2) { return new Response( JSON.stringify({ choices: [{ message: { content: '{"name":"get_price","arguments":{"coin_id":"bitcoin"}}' } }], }), { status: 200, headers: { "content-type": "application/json" } } ); } return new Response( JSON.stringify({ choices: [{ message: { content: "Bitcoin is at $64,779." } }] }), { status: 200, headers: { "content-type": "application/json" } } ); } return json({ bitcoin: { usd: 64779, usd_24h_change: 0.79, usd_market_cap: 1.3e12 } }); }; const events = []; const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) }); const degrade = events.find((e) => e.type === "degrade"); assert.ok(degrade, "the degrade must be visible in the trace"); assert.ok(seen[1] < seen[0], `retry payload must be smaller: ${seen[0]} -> ${seen[1]}`); assert.equal(seen[1], 2, "system + user only after shedding the few-shot"); assert.equal(out.answer, "Bitcoin is at $64,779."); }); test("auto mode falls back to prompted on a provider-level failure", async () => { const net = mockNetwork([ { __http: 500, message: "upstream exploded" }, textReply('{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'), textReply("Bitcoin is at $64,779."), ]); const out = await runAgent({ ...base, mode: "auto", onEvent: () => {} }); assert.equal(out.mode, "prompted"); assert.ok(out.fellBack); assert.equal(net.router.length, 3); });