/** * The agent loop. * * Two execution modes over one set of tools: * native - pass `tools` to the provider, read back `message.tool_calls` * prompted - inject schemas into the system prompt, parse calls out of text * * `auto` runs native and falls back to prompted the moment the provider shows * it cannot honour the `tools` parameter. That fallback is the whole point of * the project: it is what lets a model with no native support still call tools. * * Deliberately NOT streaming. `stream: true` combined with `tools` fragments * tool-call deltas inconsistently across providers, and the UI needs whole * calls, not slices of them. */ import { TOOL_SCHEMAS, runTool } from "./tools.js"; import { buildSystemPrompt, buildFewShot, formatToolResult, parseToolCalls, splitReasoning, coerceArgs, foldSystemIntoUser, mergeAlternating, STOP, CALL_OPEN, CALL_CLOSE, } from "./protocol.js"; const ROUTER = "https://router.huggingface.co/v1/chat/completions"; const MAX_TURNS = 6; /** Models whose chat template rejects a system role entirely. */ const NO_SYSTEM_ROLE = [/^google\/gemma-2/i, /^google\/gemma-3/i]; /** Models whose template demands strict user/assistant alternation. */ const STRICT_ALTERNATION = [/^google\/gemma/i, /^mistralai\//i]; const matches = (model, patterns) => patterns.some((p) => p.test(model)); export class InferenceError extends Error { constructor(message, { status, kind } = {}) { super(message); this.status = status; this.kind = kind; } } /* ------------------------------------------------------------------ * * transport * ------------------------------------------------------------------ */ async function callRouter({ token, model, provider, messages, tools, maxTokens, temperature, stop }) { const body = { // The router pins a provider with `model:provider`. Necessary for models // the auto-router cannot resolve on its own. model: provider ? `${model}:${provider}` : model, messages, max_tokens: maxTokens ?? 700, temperature: temperature ?? 0.2, }; if (tools) { body.tools = tools; body.tool_choice = "auto"; } if (stop) body.stop = stop; let res; try { res = await fetch(ROUTER, { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${token}` }, body: JSON.stringify(body), }); } catch (e) { throw new InferenceError(`Network error reaching the HF router: ${e.message}`, { kind: "network", }); } const text = await res.text(); const { json, embeddedError } = parseRouterBody(text); // Observed on featherless: HTTP 200 whose body is a valid empty completion // CONCATENATED with an error object. A plain JSON.parse throws on that, and // treating the result as null surfaces a misleading "no message" error. if (res.ok && embeddedError) { throw new InferenceError(`Provider failed mid-request: ${embeddedError}`, { status: 200, kind: "provider", }); } if (!res.ok) { const detail = json?.error?.message || json?.error || text.slice(0, 300); let kind = "http"; if (res.status === 401 || res.status === 403) kind = "auth"; else if (res.status === 402) kind = "credits"; else if (res.status === 429) kind = "ratelimit"; else if (/tool|function/i.test(String(detail))) kind = "tools_unsupported"; throw new InferenceError(friendlyError(res.status, detail), { status: res.status, kind }); } return json; } /** * Lenient response reader. * * Providers do not always return exactly one JSON object. Reads the first * balanced object as the payload and reports any `error` object that follows it. */ function parseRouterBody(text) { try { return { json: JSON.parse(text), embeddedError: null }; } catch { /* fall through to salvage */ } const objects = []; let depth = 0, start = -1, inStr = false, esc = false; for (let i = 0; i < text.length; i++) { const c = text[i]; if (inStr) { if (esc) esc = false; else if (c === "\\") esc = true; else if (c === '"') inStr = false; continue; } if (c === '"') inStr = true; else if (c === "{") { if (depth === 0) start = i; depth++; } else if (c === "}") { depth--; if (depth === 0 && start !== -1) { objects.push(text.slice(start, i + 1)); start = -1; } } } let json = null; let embeddedError = null; for (const raw of objects) { let obj; try { obj = JSON.parse(raw); } catch { continue; } if (obj.error) { embeddedError = obj.error.message || String(obj.error); } else if (!json) { json = obj; } } // A completion with no usable content alongside an error is a failure, even // though the transport said 200. if (json && embeddedError) { const content = json.choices?.[0]?.message?.content; if (content) embeddedError = null; } if (!json && !embeddedError) embeddedError = `unparseable response: ${text.slice(0, 200)}`; return { json, embeddedError }; } function friendlyError(status, detail) { if (status === 401 || status === 403) { return "Token rejected. Check that it is a valid HF token with Inference Providers access."; } if (status === 402) { return "HF Inference credits exhausted for this token. Sign in with your own account, or top up at huggingface.co/settings/billing."; } if (status === 429) return "Rate limited by the provider. Wait a moment and retry."; return `Provider returned HTTP ${status}: ${detail}`; } /* ------------------------------------------------------------------ * * message shaping per model capability * ------------------------------------------------------------------ */ function shapeMessages(model, messages) { let out = messages; if (matches(model, NO_SYSTEM_ROLE)) out = foldSystemIntoUser(out); if (matches(model, STRICT_ALTERNATION)) out = mergeAlternating(out); return out; } /* ------------------------------------------------------------------ * * the loop * ------------------------------------------------------------------ */ /** * @param {object} opts * @param {string} opts.userMessage * @param {Array} opts.history prior [{role, content}] visible turns * @param {string} opts.model * @param {string} opts.provider * @param {string} opts.token * @param {"auto"|"native"|"prompted"} opts.mode * @param {(e:object)=>void} opts.onEvent trace sink * @returns {Promise<{answer:string, mode:string, turns:number}>} */ export async function runAgent({ userMessage, history = [], model, provider, token, mode = "auto", onEvent = () => {}, }) { let active = mode === "auto" ? "native" : mode; let fellBack = false; for (;;) { try { const result = await runLoop({ userMessage, history, model, provider, token, mode: active, onEvent, }); return { ...result, mode: active, fellBack }; } catch (e) { const canFallBack = mode === "auto" && active === "native" && (e.kind === "tools_unsupported" || e.kind === "http" || e.kind === "provider" || e.kind === "no_native_tools"); if (!canFallBack) throw e; onEvent({ type: "fallback", reason: e.message, detail: "Native tool calling was refused by this model/provider. Retrying with the prompted protocol.", }); active = "prompted"; fellBack = true; } } } async function runLoop({ userMessage, history, model, provider, token, mode, onEvent }) { const prompted = mode === "prompted"; const messages = []; let fewShotCount = 0; if (prompted) { messages.push({ role: "system", content: buildSystemPrompt(TOOL_SCHEMAS) }); // Few-shot as real turns: the single most effective lever on whether a 2B // model emits a parseable call at all. Shed automatically if the provider // chokes on the larger payload -- see the degrade path below. const shots = buildFewShot(); fewShotCount = shots.length; messages.push(...shots); } else { messages.push({ role: "system", content: "You are a financial assistant with access to live market data tools. " + "You cannot know prices from memory: always use the tools for price, market-cap, " + "history or currency questions. Answer concisely once you have the data.", }); } messages.push(...history, { role: "user", content: userMessage }); let turn = 0; let nudged = false; let toolsRun = 0; for (turn = 1; turn <= MAX_TURNS; turn++) { onEvent({ type: "turn_start", turn }); let payload; try { payload = await callRouter({ token, model, provider, messages: shapeMessages(model, messages), tools: prompted ? undefined : TOOL_SCHEMAS, stop: prompted ? STOP : undefined, }); } catch (e) { // The few-shot block adds 10 messages. featherless has been seen failing a // request with `no_response` rather than a clean status code, and a large // payload is a plausible trigger that could not be isolated from credit // exhaustion. Shed the examples once and retry before giving up: the // shorter prompt is the configuration observed returning real content. if (e.kind === "provider" && prompted && fewShotCount > 0) { onEvent({ type: "degrade", turn, reason: e.message, detail: "Retrying without the few-shot examples, on a smaller payload.", }); messages.splice(1, fewShotCount); fewShotCount = 0; payload = await callRouter({ token, model, provider, messages: shapeMessages(model, messages), stop: STOP, }); } else { throw e; } } const msg = payload?.choices?.[0]?.message; if (!msg) throw new InferenceError("Provider returned no message.", { kind: "http" }); const usage = payload.usage; if (usage) onEvent({ type: "usage", turn, usage }); /* ---------------- native mode ---------------- */ if (!prompted) { const native = msg.tool_calls || []; // A model may ignore the native contract and emit our text protocol // anyway, or emit tags because it was fine-tuned that way. Parse both. const parsed = parseToolCalls(msg.content || "", TOOL_SCHEMAS); if (!native.length && !parsed.calls.length) { const { reasoning, text } = splitReasoning(msg.content || ""); if (reasoning) onEvent({ type: "reasoning", turn, text: reasoning }); // Turn 1 with no tool call at all is the signature of a model that // silently ignored `tools`. Escalate so `auto` can fall back. if (turn === 1 && needsTools(userMessage)) { throw new InferenceError( "Model returned prose instead of a tool call on the first turn.", { kind: "no_native_tools" } ); } onEvent({ type: "final", turn, text }); return { answer: text, turns: turn }; } const calls = native.length ? native.map((c) => ({ id: c.id, name: c.function?.name, arguments: safeArgs(c.function?.arguments), })) : parsed.calls.map((c, i) => ({ id: `parsed_${turn}_${i}`, ...c })); if (msg.content) { const { reasoning, text } = splitReasoning(msg.content); if (reasoning) onEvent({ type: "reasoning", turn, text: reasoning }); if (text) onEvent({ type: "thought", turn, text }); } const results = await executeAll(calls, turn, onEvent); toolsRun += results.length; messages.push({ role: "assistant", content: msg.content || null, tool_calls: calls.map((c) => ({ id: c.id, type: "function", function: { name: c.name, arguments: JSON.stringify(c.arguments) }, })), }); for (const r of results) { messages.push({ role: "tool", tool_call_id: r.id, content: JSON.stringify(r.result) }); } continue; } /* ---------------- prompted mode ---------------- */ const raw = msg.content || ""; const { reasoning, calls, text } = parseToolCalls(raw, TOOL_SCHEMAS); if (reasoning) onEvent({ type: "reasoning", turn, text: reasoning }); if (!calls.length) { // The characteristic small-model failure: it narrates its intention // ("I need the live price of Bitcoin.") and stops without emitting the // call. Observed repeatedly on Qwen3.5-2B. One explicit nudge recovers it // far more often than a longer system prompt does. // Only when nothing has been fetched yet. Once a tool has run, a prose // reply is the answer we asked for, not a failure to call. if (!nudged && toolsRun === 0 && needsTools(userMessage)) { nudged = true; onEvent({ type: "nudge", turn, text: text || "(empty reply)", detail: "No tool call emitted. Re-prompting with an explicit format reminder.", }); messages.push({ role: "assistant", content: text || "..." }); messages.push({ role: "user", content: `You did not emit a tool call, so no data was retrieved. Do not describe ` + `what you intend to do. Reply with ONLY the tool call block and nothing ` + `else, starting at the very first character:\n` + `${CALL_OPEN}{"name": "...", "arguments": {...}}${CALL_CLOSE}`, }); continue; } onEvent({ type: "final", turn, text }); return { answer: text, turns: turn }; } if (text) onEvent({ type: "thought", turn, text }); const withIds = calls.map((c, i) => ({ id: `p_${turn}_${i}`, ...c })); const results = await executeAll(withIds, turn, onEvent); toolsRun += results.length; // Echo the model's own call text back as its assistant turn, then deliver // results as a USER turn. Keeps user/assistant strictly alternating, which // templates without a tool role require. messages.push({ role: "assistant", content: rebuildAssistantTurn(text, withIds) }); messages.push({ role: "user", content: results.map((r) => formatToolResult(r.name, r.result)).join("\n"), }); } onEvent({ type: "final", turn, text: "I reached the maximum number of tool-calling turns without settling on an answer. " + "Try rephrasing, or ask for one thing at a time.", }); return { answer: "Turn limit reached.", turns: MAX_TURNS }; } /* ------------------------------------------------------------------ * * helpers * ------------------------------------------------------------------ */ function safeArgs(raw) { if (!raw) return {}; if (typeof raw === "object") return raw; try { return JSON.parse(raw); } catch { return {}; } } /** Heuristic: does this question actually require live data? * Used only to decide whether a tool-less first turn is suspicious. */ function needsTools(q) { return /price|worth|cost|market|cap|convert|trend|change|how much|vs\.?|compare|rate|usd|eur|try|btc|eth/i.test( q ); } function rebuildAssistantTurn(text, calls) { const blocks = calls .map((c) => `${JSON.stringify({ name: c.name, arguments: c.arguments })}`) .join("\n"); return text ? `${text}\n${blocks}` : blocks; } async function executeAll(calls, turn, onEvent) { return Promise.all( calls.map(async (c) => { const args = coerceArgs(c.name, c.arguments || {}, TOOL_SCHEMAS); onEvent({ type: "tool_call", turn, name: c.name, args }); const started = Date.now(); const result = await runTool(c.name, args); onEvent({ type: "tool_result", turn, name: c.name, result, ms: Date.now() - started, failed: !!result?.error, }); return { id: c.id, name: c.name, result }; }) ); }