agent.js

/**
 * The agent loop.
 *
 * Two execution modes over one set of tools:
 *   native   - pass `tools` to the provider, read back `message.tool_calls`
 *   prompted - inject schemas into the system prompt, parse calls out of text
 *
 * `auto` runs native and falls back to prompted the moment the provider shows
 * it cannot honour the `tools` parameter. That fallback is the whole point of
 * the project: it is what lets a model with no native support still call tools.
 *
 * Deliberately NOT streaming. `stream: true` combined with `tools` fragments
 * tool-call deltas inconsistently across providers, and the UI needs whole
 * calls, not slices of them.
 */

import { TOOL_SCHEMAS, runTool } from "./tools.js";
import {
  buildSystemPrompt,
  buildFewShot,
  formatToolResult,
  parseToolCalls,
  splitReasoning,
  coerceArgs,
  foldSystemIntoUser,
  mergeAlternating,
  STOP,
  CALL_OPEN,
  CALL_CLOSE,
} from "./protocol.js";

const ROUTER = "https://router.huggingface.co/v1/chat/completions";
const MAX_TURNS = 6;

/** Models whose chat template rejects a system role entirely. */
const NO_SYSTEM_ROLE = [/^google\/gemma-2/i, /^google\/gemma-3/i];

/** Models whose template demands strict user/assistant alternation. */
const STRICT_ALTERNATION = [/^google\/gemma/i, /^mistralai\//i];

const matches = (model, patterns) => patterns.some((p) => p.test(model));

export class InferenceError extends Error {
  constructor(message, { status, kind } = {}) {
    super(message);
    this.status = status;
    this.kind = kind;
  }
}

/* ------------------------------------------------------------------ *
 * transport
 * ------------------------------------------------------------------ */

async function callRouter({ token, model, provider, messages, tools, maxTokens, temperature, stop }) {
  const body = {
    // The router pins a provider with `model:provider`. Necessary for models
    // the auto-router cannot resolve on its own.
    model: provider ? `${model}:${provider}` : model,
    messages,
    max_tokens: maxTokens ?? 700,
    temperature: temperature ?? 0.2,
  };
  if (tools) {
    body.tools = tools;
    body.tool_choice = "auto";
  }
  if (stop) body.stop = stop;

  let res;
  try {
    res = await fetch(ROUTER, {
      method: "POST",
      headers: { "content-type": "application/json", authorization: `Bearer ${token}` },
      body: JSON.stringify(body),
    });
  } catch (e) {
    throw new InferenceError(`Network error reaching the HF router: ${e.message}`, {
      kind: "network",
    });
  }

  const text = await res.text();
  const { json, embeddedError } = parseRouterBody(text);

  // Observed on featherless: HTTP 200 whose body is a valid empty completion
  // CONCATENATED with an error object. A plain JSON.parse throws on that, and
  // treating the result as null surfaces a misleading "no message" error.
  if (res.ok && embeddedError) {
    throw new InferenceError(`Provider failed mid-request: ${embeddedError}`, {
      status: 200,
      kind: "provider",
    });
  }

  if (!res.ok) {
    const detail = json?.error?.message || json?.error || text.slice(0, 300);
    let kind = "http";
    if (res.status === 401 || res.status === 403) kind = "auth";
    else if (res.status === 402) kind = "credits";
    else if (res.status === 429) kind = "ratelimit";
    else if (/tool|function/i.test(String(detail))) kind = "tools_unsupported";
    throw new InferenceError(friendlyError(res.status, detail), { status: res.status, kind });
  }

  return json;
}

/**
 * Lenient response reader.
 *
 * Providers do not always return exactly one JSON object. Reads the first
 * balanced object as the payload and reports any `error` object that follows it.
 */
function parseRouterBody(text) {
  try {
    return { json: JSON.parse(text), embeddedError: null };
  } catch {
    /* fall through to salvage */
  }

  const objects = [];
  let depth = 0,
    start = -1,
    inStr = false,
    esc = false;
  for (let i = 0; i < text.length; i++) {
    const c = text[i];
    if (inStr) {
      if (esc) esc = false;
      else if (c === "\\") esc = true;
      else if (c === '"') inStr = false;
      continue;
    }
    if (c === '"') inStr = true;
    else if (c === "{") {
      if (depth === 0) start = i;
      depth++;
    } else if (c === "}") {
      depth--;
      if (depth === 0 && start !== -1) {
        objects.push(text.slice(start, i + 1));
        start = -1;
      }
    }
  }

  let json = null;
  let embeddedError = null;
  for (const raw of objects) {
    let obj;
    try {
      obj = JSON.parse(raw);
    } catch {
      continue;
    }
    if (obj.error) {
      embeddedError = obj.error.message || String(obj.error);
    } else if (!json) {
      json = obj;
    }
  }

  // A completion with no usable content alongside an error is a failure, even
  // though the transport said 200.
  if (json && embeddedError) {
    const content = json.choices?.[0]?.message?.content;
    if (content) embeddedError = null;
  }

  if (!json && !embeddedError) embeddedError = `unparseable response: ${text.slice(0, 200)}`;
  return { json, embeddedError };
}

function friendlyError(status, detail) {
  if (status === 401 || status === 403) {
    return "Token rejected. Check that it is a valid HF token with Inference Providers access.";
  }
  if (status === 402) {
    return "HF Inference credits exhausted for this token. Sign in with your own account, or top up at huggingface.co/settings/billing.";
  }
  if (status === 429) return "Rate limited by the provider. Wait a moment and retry.";
  return `Provider returned HTTP ${status}: ${detail}`;
}

/* ------------------------------------------------------------------ *
 * message shaping per model capability
 * ------------------------------------------------------------------ */

function shapeMessages(model, messages) {
  let out = messages;
  if (matches(model, NO_SYSTEM_ROLE)) out = foldSystemIntoUser(out);
  if (matches(model, STRICT_ALTERNATION)) out = mergeAlternating(out);
  return out;
}

/* ------------------------------------------------------------------ *
 * the loop
 * ------------------------------------------------------------------ */

/**
 * @param {object}   opts
 * @param {string}   opts.userMessage
 * @param {Array}    opts.history      prior [{role, content}] visible turns
 * @param {string}   opts.model
 * @param {string}   opts.provider
 * @param {string}   opts.token
 * @param {"auto"|"native"|"prompted"} opts.mode
 * @param {(e:object)=>void} opts.onEvent  trace sink
 * @returns {Promise<{answer:string, mode:string, turns:number}>}
 */
export async function runAgent({
  userMessage,
  history = [],
  model,
  provider,
  token,
  mode = "auto",
  onEvent = () => {},
}) {
  let active = mode === "auto" ? "native" : mode;
  let fellBack = false;

  for (;;) {
    try {
      const result = await runLoop({
        userMessage,
        history,
        model,
        provider,
        token,
        mode: active,
        onEvent,
      });
      return { ...result, mode: active, fellBack };
    } catch (e) {
      const canFallBack =
        mode === "auto" &&
        active === "native" &&
        (e.kind === "tools_unsupported" ||
          e.kind === "http" ||
          e.kind === "provider" ||
          e.kind === "no_native_tools");

      if (!canFallBack) throw e;

      onEvent({
        type: "fallback",
        reason: e.message,
        detail:
          "Native tool calling was refused by this model/provider. Retrying with the prompted protocol.",
      });
      active = "prompted";
      fellBack = true;
    }
  }
}

async function runLoop({ userMessage, history, model, provider, token, mode, onEvent }) {
  const prompted = mode === "prompted";

  const messages = [];
  let fewShotCount = 0;
  if (prompted) {
    messages.push({ role: "system", content: buildSystemPrompt(TOOL_SCHEMAS) });
    // Few-shot as real turns: the single most effective lever on whether a 2B
    // model emits a parseable call at all. Shed automatically if the provider
    // chokes on the larger payload -- see the degrade path below.
    const shots = buildFewShot();
    fewShotCount = shots.length;
    messages.push(...shots);
  } else {
    messages.push({
      role: "system",
      content:
        "You are a financial assistant with access to live market data tools. " +
        "You cannot know prices from memory: always use the tools for price, market-cap, " +
        "history or currency questions. Answer concisely once you have the data.",
    });
  }
  messages.push(...history, { role: "user", content: userMessage });

  let turn = 0;
  let nudged = false;
  let toolsRun = 0;

  for (turn = 1; turn <= MAX_TURNS; turn++) {
    onEvent({ type: "turn_start", turn });

    let payload;
    try {
      payload = await callRouter({
        token,
        model,
        provider,
        messages: shapeMessages(model, messages),
        tools: prompted ? undefined : TOOL_SCHEMAS,
        stop: prompted ? STOP : undefined,
      });
    } catch (e) {
      // The few-shot block adds 10 messages. featherless has been seen failing a
      // request with `no_response` rather than a clean status code, and a large
      // payload is a plausible trigger that could not be isolated from credit
      // exhaustion. Shed the examples once and retry before giving up: the
      // shorter prompt is the configuration observed returning real content.
      if (e.kind === "provider" && prompted && fewShotCount > 0) {
        onEvent({
          type: "degrade",
          turn,
          reason: e.message,
          detail: "Retrying without the few-shot examples, on a smaller payload.",
        });
        messages.splice(1, fewShotCount);
        fewShotCount = 0;
        payload = await callRouter({
          token,
          model,
          provider,
          messages: shapeMessages(model, messages),
          stop: STOP,
        });
      } else {
        throw e;
      }
    }

    const msg = payload?.choices?.[0]?.message;
    if (!msg) throw new InferenceError("Provider returned no message.", { kind: "http" });

    const usage = payload.usage;
    if (usage) onEvent({ type: "usage", turn, usage });

    /* ---------------- native mode ---------------- */
    if (!prompted) {
      const native = msg.tool_calls || [];

      // A model may ignore the native contract and emit our text protocol
      // anyway, or emit tags because it was fine-tuned that way. Parse both.
      const parsed = parseToolCalls(msg.content || "", TOOL_SCHEMAS);

      if (!native.length && !parsed.calls.length) {
        const { reasoning, text } = splitReasoning(msg.content || "");
        if (reasoning) onEvent({ type: "reasoning", turn, text: reasoning });

        // Turn 1 with no tool call at all is the signature of a model that
        // silently ignored `tools`. Escalate so `auto` can fall back.
        if (turn === 1 && needsTools(userMessage)) {
          throw new InferenceError(
            "Model returned prose instead of a tool call on the first turn.",
            { kind: "no_native_tools" }
          );
        }
        onEvent({ type: "final", turn, text });
        return { answer: text, turns: turn };
      }

      const calls = native.length
        ? native.map((c) => ({
            id: c.id,
            name: c.function?.name,
            arguments: safeArgs(c.function?.arguments),
          }))
        : parsed.calls.map((c, i) => ({ id: `parsed_${turn}_${i}`, ...c }));

      if (msg.content) {
        const { reasoning, text } = splitReasoning(msg.content);
        if (reasoning) onEvent({ type: "reasoning", turn, text: reasoning });
        if (text) onEvent({ type: "thought", turn, text });
      }

      const results = await executeAll(calls, turn, onEvent);
      toolsRun += results.length;

      messages.push({
        role: "assistant",
        content: msg.content || null,
        tool_calls: calls.map((c) => ({
          id: c.id,
          type: "function",
          function: { name: c.name, arguments: JSON.stringify(c.arguments) },
        })),
      });
      for (const r of results) {
        messages.push({ role: "tool", tool_call_id: r.id, content: JSON.stringify(r.result) });
      }
      continue;
    }

    /* ---------------- prompted mode ---------------- */
    const raw = msg.content || "";
    const { reasoning, calls, text } = parseToolCalls(raw, TOOL_SCHEMAS);

    if (reasoning) onEvent({ type: "reasoning", turn, text: reasoning });

    if (!calls.length) {
      // The characteristic small-model failure: it narrates its intention
      // ("I need the live price of Bitcoin.") and stops without emitting the
      // call. Observed repeatedly on Qwen3.5-2B. One explicit nudge recovers it
      // far more often than a longer system prompt does.
      // Only when nothing has been fetched yet. Once a tool has run, a prose
      // reply is the answer we asked for, not a failure to call.
      if (!nudged && toolsRun === 0 && needsTools(userMessage)) {
        nudged = true;
        onEvent({
          type: "nudge",
          turn,
          text: text || "(empty reply)",
          detail: "No tool call emitted. Re-prompting with an explicit format reminder.",
        });
        messages.push({ role: "assistant", content: text || "..." });
        messages.push({
          role: "user",
          content:
            `You did not emit a tool call, so no data was retrieved. Do not describe ` +
            `what you intend to do. Reply with ONLY the tool call block and nothing ` +
            `else, starting at the very first character:\n` +
            `${CALL_OPEN}{"name": "...", "arguments": {...}}${CALL_CLOSE}`,
        });
        continue;
      }

      onEvent({ type: "final", turn, text });
      return { answer: text, turns: turn };
    }

    if (text) onEvent({ type: "thought", turn, text });

    const withIds = calls.map((c, i) => ({ id: `p_${turn}_${i}`, ...c }));
    const results = await executeAll(withIds, turn, onEvent);
    toolsRun += results.length;

    // Echo the model's own call text back as its assistant turn, then deliver
    // results as a USER turn. Keeps user/assistant strictly alternating, which
    // templates without a tool role require.
    messages.push({ role: "assistant", content: rebuildAssistantTurn(text, withIds) });
    messages.push({
      role: "user",
      content: results.map((r) => formatToolResult(r.name, r.result)).join("\n"),
    });
  }

  onEvent({
    type: "final",
    turn,
    text:
      "I reached the maximum number of tool-calling turns without settling on an answer. " +
      "Try rephrasing, or ask for one thing at a time.",
  });
  return { answer: "Turn limit reached.", turns: MAX_TURNS };
}

/* ------------------------------------------------------------------ *
 * helpers
 * ------------------------------------------------------------------ */

function safeArgs(raw) {
  if (!raw) return {};
  if (typeof raw === "object") return raw;
  try {
    return JSON.parse(raw);
  } catch {
    return {};
  }
}

/** Heuristic: does this question actually require live data?
 *  Used only to decide whether a tool-less first turn is suspicious. */
function needsTools(q) {
  return /price|worth|cost|market|cap|convert|trend|change|how much|vs\.?|compare|rate|usd|eur|try|btc|eth/i.test(
    q
  );
}

function rebuildAssistantTurn(text, calls) {
  const blocks = calls
    .map((c) => `<tool_call>${JSON.stringify({ name: c.name, arguments: c.arguments })}</tool_call>`)
    .join("\n");
  return text ? `${text}\n${blocks}` : blocks;
}

async function executeAll(calls, turn, onEvent) {
  return Promise.all(
    calls.map(async (c) => {
      const args = coerceArgs(c.name, c.arguments || {}, TOOL_SCHEMAS);
      onEvent({ type: "tool_call", turn, name: c.name, args });
      const started = Date.now();
      const result = await runTool(c.name, args);
      onEvent({
        type: "tool_result",
        turn,
        name: c.name,
        result,
        ms: Date.now() - started,
        failed: !!result?.error,
      });
      return { id: c.id, name: c.name, result };
    })
  );
}