tests/agent.test.mjs

/**
 * Agent-loop tests with a fully mocked network. No credits, no live APIs.
 *
 * Both the HF router and the data APIs are intercepted, so these exercise the
 * real control flow: native path, prompted path, the auto fallback between
 * them, multi-turn chaining and error recovery.
 */
import { test } from "node:test";
import assert from "node:assert/strict";
import { runAgent } from "../agent.js";

const realFetch = globalThis.fetch;

/**
 * @param {Array<Function|object>} routerReplies  one per router call, in order.
 *   A function receives the parsed request body; an object is returned as-is.
 */
function mockNetwork(routerReplies) {
  const calls = { router: [], data: [] };
  let i = 0;

  globalThis.fetch = async (url, init = {}) => {
    const u = String(url);

    if (u.includes("router.huggingface.co")) {
      const body = JSON.parse(init.body);
      calls.router.push(body);
      const reply = routerReplies[i++];
      if (reply === undefined) throw new Error(`unexpected router call #${i}`);
      const resolved = typeof reply === "function" ? reply(body) : reply;
      if (resolved.__http) {
        return new Response(JSON.stringify({ error: { message: resolved.message } }), {
          status: resolved.__http,
          headers: { "content-type": "application/json" },
        });
      }
      return new Response(JSON.stringify(resolved), {
        status: 200,
        headers: { "content-type": "application/json" },
      });
    }

    calls.data.push(u);
    if (u.includes("/simple/price")) {
      return json({ bitcoin: { usd: 64779, usd_24h_change: 0.79, usd_market_cap: 1.3e12 } });
    }
    if (u.includes("/search")) {
      return json({ coins: [{ id: "bitcoin", symbol: "btc", name: "Bitcoin", market_cap_rank: 1 }] });
    }
    if (u.includes("open.er-api.com")) {
      return json({ result: "success", rates: { TRY: 42.5 }, time_last_update_utc: "now" });
    }
    return json({});
  };

  return calls;
}

const json = (o) =>
  new Response(JSON.stringify(o), { status: 200, headers: { "content-type": "application/json" } });

const nativeReply = (toolCalls, content = null) => ({
  choices: [
    {
      message: {
        role: "assistant",
        content,
        tool_calls: toolCalls.map((c, i) => ({
          id: `call_${i}`,
          type: "function",
          function: { name: c.name, arguments: JSON.stringify(c.arguments) },
        })),
      },
    },
  ],
  usage: { prompt_tokens: 10, completion_tokens: 5 },
});

const textReply = (content) => ({
  choices: [{ message: { role: "assistant", content } }],
  usage: { prompt_tokens: 10, completion_tokens: 5 },
});

const base = { model: "test/model", token: "hf_test", userMessage: "What is the price of bitcoin?" };

test.afterEach(() => {
  globalThis.fetch = realFetch;
});

/* ---------------------------------------------------------------- */

test("native mode: tool call, then final answer", async () => {
  const net = mockNetwork([
    nativeReply([{ name: "get_price", arguments: { coin_id: "bitcoin" } }]),
    textReply("Bitcoin is at $64,779."),
  ]);
  const events = [];
  const out = await runAgent({ ...base, mode: "native", onEvent: (e) => events.push(e) });

  assert.equal(out.answer, "Bitcoin is at $64,779.");
  assert.equal(out.turns, 2);
  assert.equal(events.filter((e) => e.type === "tool_call").length, 1);
  assert.equal(events.find((e) => e.type === "tool_result").result.price, 64779);

  // Second request must carry the assistant tool_calls turn and a tool result.
  const second = net.router[1];
  assert.ok(second.messages.some((m) => m.role === "assistant" && m.tool_calls));
  assert.ok(second.messages.some((m) => m.role === "tool"));
  assert.ok(second.tools, "native mode must send the tools array");
});

test("prompted mode: parses a tagged call and feeds the result back as a USER turn", async () => {
  const net = mockNetwork([
    textReply('I need the price.\n<tool_call>{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'),
    textReply("Bitcoin is at $64,779."),
  ]);
  const out = await runAgent({ ...base, mode: "prompted", onEvent: () => {} });

  assert.equal(out.answer, "Bitcoin is at $64,779.");

  const first = net.router[0];
  assert.ok(!first.tools, "prompted mode must NOT send a tools array");
  assert.deepEqual(first.stop, ["</tool_call>"], "stop sequence is required");
  assert.match(first.messages[0].content, /AVAILABLE TOOLS/);

  const second = net.router[1];
  const fed = second.messages[second.messages.length - 1];
  assert.equal(fed.role, "user", "results must come back as a user turn, not role:tool");
  assert.match(fed.content, /^TOOL_RESULT get_price /);
  assert.ok(!second.messages.some((m) => m.role === "tool"), "no tool role in prompted mode");
});

test("auto mode falls back to prompted when the provider rejects tools", async () => {
  const net = mockNetwork([
    { __http: 400, message: "tools are not supported for this model" },
    textReply('<tool_call>{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'),
    textReply("Bitcoin is at $64,779."),
  ]);
  const events = [];
  const out = await runAgent({ ...base, mode: "auto", onEvent: (e) => events.push(e) });

  assert.equal(out.mode, "prompted");
  assert.ok(out.fellBack);
  assert.ok(events.some((e) => e.type === "fallback"), "the fallback must be visible in the trace");
  assert.ok(!net.router[0].stop, "the first (native) attempt sends no stop sequence");
  assert.deepEqual(net.router[1].stop, ["</tool_call>"]);
  assert.equal(out.answer, "Bitcoin is at $64,779.");
});

test("auto mode falls back when the model silently ignores `tools` and returns prose", async () => {
  // The dangerous case: HTTP 200, no tool_calls, just an invented answer.
  const net = mockNetwork([
    textReply("Bitcoin is worth about twenty thousand dollars, I think."),
    textReply('<tool_call>{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'),
    textReply("Bitcoin is at $64,779."),
  ]);
  const out = await runAgent({ ...base, mode: "auto", onEvent: () => {} });

  assert.equal(out.mode, "prompted");
  assert.equal(out.answer, "Bitcoin is at $64,779.");
  assert.equal(net.router.length, 3);
});

test("auto does NOT fall back for a genuine non-tool question", async () => {
  const net = mockNetwork([textReply("Hello! How can I help with market data today?")]);
  const out = await runAgent({
    ...base,
    userMessage: "hello there",
    mode: "auto",
    onEvent: () => {},
  });
  assert.equal(out.mode, "native");
  assert.equal(net.router.length, 1, "a greeting must not trigger a fallback retry");
});

test("multi-turn chaining: price, then currency conversion", async () => {
  mockNetwork([
    textReply('<tool_call>{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'),
    textReply('<tool_call>{"name":"convert_currency","arguments":{"amount":64779,"from_currency":"USD","to_currency":"TRY"}}'),
    textReply("Bitcoin is $64,779, about 2,753,107 TRY."),
  ]);
  const events = [];
  const out = await runAgent({
    ...base,
    userMessage: "What is bitcoin in Turkish lira?",
    mode: "prompted",
    onEvent: (e) => events.push(e),
  });

  const names = events.filter((e) => e.type === "tool_call").map((e) => e.name);
  assert.deepEqual(names, ["get_price", "convert_currency"]);
  assert.equal(out.turns, 3);
  const conv = events.filter((e) => e.type === "tool_result").at(-1);
  assert.equal(conv.result.result, 2753107.5);
});

test("two parallel calls in one turn are both executed", async () => {
  mockNetwork([
    textReply(
      '<tool_call>{"name":"get_price","arguments":{"coin_id":"bitcoin"}}</tool_call>\n' +
        '<tool_call>{"name":"search_coin","arguments":{"query":"btc"}}</tool_call>'
    ),
    textReply("Done."),
  ]);
  const events = [];
  await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) });
  assert.equal(events.filter((e) => e.type === "tool_call").length, 2);
});

test("a tool error is fed back as data so the model can recover", async () => {
  mockNetwork([
    textReply('<tool_call>{"name":"no_such_tool","arguments":{}}'),
    textReply("Sorry, that tool does not exist."),
  ]);
  const events = [];
  const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) });
  const res = events.find((e) => e.type === "tool_result");
  assert.ok(res.failed);
  assert.match(res.result.error, /unknown tool/);
  assert.equal(out.answer, "Sorry, that tool does not exist.");
});

test("gemma-2: system role is folded away and turns strictly alternate", async () => {
  const net = mockNetwork([textReply("Hi there.")]);
  await runAgent({
    ...base,
    model: "google/gemma-2-2b-it",
    userMessage: "hello",
    mode: "prompted",
    onEvent: () => {},
  });
  const msgs = net.router[0].messages;
  assert.ok(!msgs.some((m) => m.role === "system"), "gemma-2 rejects a system role");
  assert.match(msgs[0].content, /AVAILABLE TOOLS/, "the prompt must survive the fold");
  for (let i = 1; i < msgs.length; i++) {
    assert.notEqual(msgs[i].role, msgs[i - 1].role, "gemma-2 needs strict alternation");
  }
});

test("credit exhaustion surfaces a clear message and does not retry", async () => {
  const net = mockNetwork([{ __http: 402, message: "Payment Required" }]);
  await assert.rejects(
    () => runAgent({ ...base, mode: "auto", onEvent: () => {} }),
    (e) => {
      assert.equal(e.kind, "credits");
      assert.match(e.message, /credits exhausted/i);
      return true;
    }
  );
  assert.equal(net.router.length, 1, "402 must not trigger a prompted retry");
});

test("a bad token surfaces an auth error, not a fallback", async () => {
  const net = mockNetwork([{ __http: 401, message: "Invalid credentials" }]);
  await assert.rejects(
    () => runAgent({ ...base, mode: "auto", onEvent: () => {} }),
    (e) => {
      assert.equal(e.kind, "auth");
      assert.match(e.message, /Token rejected/);
      return true;
    }
  );
  assert.equal(net.router.length, 1, "401 must not trigger a prompted retry");
});

test("provider is pinned onto the model id as model:provider", async () => {
  const net = mockNetwork([textReply("hi")]);
  await runAgent({
    ...base,
    userMessage: "hello",
    provider: "featherless-ai",
    mode: "prompted",
    onEvent: () => {},
  });
  assert.equal(net.router[0].model, "test/model:featherless-ai");
});

test("the turn cap stops a model that loops forever", async () => {
  const looping = Array.from({ length: 8 }, () =>
    textReply('<tool_call>{"name":"get_price","arguments":{"coin_id":"bitcoin"}}')
  );
  mockNetwork(looping);
  const events = [];
  const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) });
  assert.equal(out.turns, 6);
  assert.match(events.at(-1).text, /maximum number of tool-calling turns/);
});

/* ---------------------------------------------------------------- *
 * regressions found during live testing against Qwen3.5-2B
 * ---------------------------------------------------------------- */

test("REGRESSION: concatenated completion+error body is reported, not swallowed", async () => {
  // Real featherless response: HTTP 200, a valid empty completion, then an
  // error object appended. JSON.parse throws on this.
  globalThis.fetch = async () =>
    new Response(
      JSON.stringify({ choices: [{ message: { role: "assistant", content: "" } }] }) +
        JSON.stringify({
          error: { message: "No successful response received from completion service" },
        }),
      { status: 200, headers: { "content-type": "application/json" } }
    );

  await assert.rejects(
    () => runAgent({ ...base, mode: "prompted", onEvent: () => {} }),
    (e) => {
      assert.equal(e.kind, "provider");
      assert.match(e.message, /No successful response received/);
      return true;
    }
  );
});

test("a concatenated body WITH real content is still usable", async () => {
  globalThis.fetch = async () =>
    new Response(
      JSON.stringify({ choices: [{ message: { role: "assistant", content: "Bitcoin is $64,779." } }] }) +
        JSON.stringify({ error: { message: "late warning" } }),
      { status: 200, headers: { "content-type": "application/json" } }
    );
  const out = await runAgent({ ...base, mode: "prompted", onEvent: () => {} });
  assert.equal(out.answer, "Bitcoin is $64,779.");
});

test("REGRESSION: prose-instead-of-call is nudged, then recovers", async () => {
  // The exact captured Qwen3.5-2B failure: it narrates intent and stops.
  const net = mockNetwork([
    textReply("I need the live price of Bitcoin."),
    textReply('<tool_call>{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'),
    textReply("Bitcoin is at $64,779."),
  ]);
  const events = [];
  const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) });

  const nudge = events.find((e) => e.type === "nudge");
  assert.ok(nudge, "the nudge must be visible in the trace");
  assert.match(nudge.text, /I need the live price/);
  assert.equal(out.answer, "Bitcoin is at $64,779.");

  const retry = net.router[1];
  const last = retry.messages.at(-1);
  assert.equal(last.role, "user");
  assert.match(last.content, /ONLY the tool call block/);
});

test("the nudge fires at most once, then accepts the prose as final", async () => {
  const net = mockNetwork([
    textReply("I need the live price of Bitcoin."),
    textReply("I really do need it."),
  ]);
  const events = [];
  const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) });
  assert.equal(events.filter((e) => e.type === "nudge").length, 1, "must not nudge repeatedly");
  assert.equal(net.router.length, 2);
  assert.equal(out.answer, "I really do need it.");
});

test("a non-tool question is never nudged", async () => {
  const net = mockNetwork([textReply("Hello, how can I help?")]);
  await runAgent({ ...base, userMessage: "hello", mode: "prompted", onEvent: () => {} });
  assert.equal(net.router.length, 1);
});

test("few-shot examples are delivered as real alternating message turns", async () => {
  const net = mockNetwork([textReply("hi")]);
  await runAgent({ ...base, userMessage: "hello", mode: "prompted", onEvent: () => {} });
  const msgs = net.router[0].messages;
  assert.equal(msgs[0].role, "system");
  // The demonstrated assistant turns must contain ONLY the call: a leading prose
  // line is what made Qwen3.5-2B stop before emitting one.
  const shots = msgs.filter((m) => m.role === "assistant" && m.content.includes("<tool_call>"));
  assert.ok(shots.length >= 2, "expected few-shot assistant turns with calls");
  for (const s of shots) {
    assert.ok(s.content.trimStart().startsWith("<tool_call>"), `prose before call: ${s.content}`);
  }
  for (let i = 1; i < msgs.length; i++) {
    if (msgs[i].role !== "system") assert.notEqual(msgs[i].role, msgs[i - 1].role);
  }
});

test("prompted mode sheds the few-shot and retries when the provider errors", async () => {
  let n = 0;
  const seen = [];
  globalThis.fetch = async (url, init = {}) => {
    const u = String(url);
    if (u.includes("router.huggingface.co")) {
      const body = JSON.parse(init.body);
      seen.push(body.messages.length);
      n++;
      if (n === 1) {
        // The featherless shape: 200, empty completion, trailing error object.
        return new Response(
          JSON.stringify({ choices: [{ message: { role: "assistant", content: "" } }] }) +
            JSON.stringify({ error: { message: "No successful response received" } }),
          { status: 200, headers: { "content-type": "application/json" } }
        );
      }
      if (n === 2) {
        return new Response(
          JSON.stringify({
            choices: [{ message: { content: '<tool_call>{"name":"get_price","arguments":{"coin_id":"bitcoin"}}' } }],
          }),
          { status: 200, headers: { "content-type": "application/json" } }
        );
      }
      return new Response(
        JSON.stringify({ choices: [{ message: { content: "Bitcoin is at $64,779." } }] }),
        { status: 200, headers: { "content-type": "application/json" } }
      );
    }
    return json({ bitcoin: { usd: 64779, usd_24h_change: 0.79, usd_market_cap: 1.3e12 } });
  };

  const events = [];
  const out = await runAgent({ ...base, mode: "prompted", onEvent: (e) => events.push(e) });

  const degrade = events.find((e) => e.type === "degrade");
  assert.ok(degrade, "the degrade must be visible in the trace");
  assert.ok(seen[1] < seen[0], `retry payload must be smaller: ${seen[0]} -> ${seen[1]}`);
  assert.equal(seen[1], 2, "system + user only after shedding the few-shot");
  assert.equal(out.answer, "Bitcoin is at $64,779.");
});

test("auto mode falls back to prompted on a provider-level failure", async () => {
  const net = mockNetwork([
    { __http: 500, message: "upstream exploded" },
    textReply('<tool_call>{"name":"get_price","arguments":{"coin_id":"bitcoin"}}'),
    textReply("Bitcoin is at $64,779."),
  ]);
  const out = await runAgent({ ...base, mode: "auto", onEvent: () => {} });
  assert.equal(out.mode, "prompted");
  assert.ok(out.fellBack);
  assert.equal(net.router.length, 3);
});