// Remote model backend: any OpenAI-compatible Chat Completions API (OpenAI, OpenRouter, // the Hugging Face router, Groq, a local Ollama/LM Studio server, ...). Used when the // local model can't run on this device. The Linux machine still runs in the tab. // // It has the same interface as LLM (llm.js), so the agent loop is unchanged: the agent's // history (LFM3 text format) is translated into API messages with native tool calls, and // the streamed reply is rendered back into that text format for the agent to parse. import { parseAssistant } from "./toolcalls.js"; // Providers with an OpenAI-compatible Chat Completions API that accept requests from web // pages (CORS preflight checked for each). `avatar` is the provider's Hugging Face avatar // in vendor/avatars/ (none for "Other"). export const API_PRESETS = [ { id: "huggingface", signIn: true, defaultModel: "openai/gpt-oss-120b", label: "Hugging Face", baseUrl: "https://router.huggingface.co/v1", avatar: "huggingface", keysUrl: "https://huggingface.co/settings/tokens", hint: "Open models from many inference providers, billed to your Hugging Face account." }, { id: "openrouter", label: "OpenRouter", baseUrl: "https://openrouter.ai/api/v1", avatar: "openrouter", keysUrl: "https://openrouter.ai/keys", hint: "Hundreds of models from every major lab behind one key." }, { id: "openai", label: "OpenAI", baseUrl: "https://api.openai.com/v1", avatar: "openai", keysUrl: "https://platform.openai.com/api-keys" }, { id: "anthropic", label: "Anthropic", baseUrl: "https://api.anthropic.com/v1", avatar: "anthropic", keysUrl: "https://console.anthropic.com/settings/keys", headers: { "anthropic-dangerous-direct-browser-access": "true" }, hint: "Uses Anthropic's OpenAI-compatible endpoint." }, { id: "gemini", label: "Google Gemini", baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai", avatar: "google", keysUrl: "https://aistudio.google.com/apikey" }, { id: "groq", label: "Groq", baseUrl: "https://api.groq.com/openai/v1", avatar: "groq", keysUrl: "https://console.groq.com/keys" }, { id: "mistral", label: "Mistral AI", baseUrl: "https://api.mistral.ai/v1", avatar: "mistral", keysUrl: "https://console.mistral.ai/api-keys" }, { id: "deepseek", label: "DeepSeek", baseUrl: "https://api.deepseek.com/v1", avatar: "deepseek", keysUrl: "https://platform.deepseek.com/api_keys" }, { id: "xai", label: "xAI", baseUrl: "https://api.x.ai/v1", avatar: "xai", keysUrl: "https://console.x.ai" }, { id: "together", label: "Together AI", baseUrl: "https://api.together.xyz/v1", avatar: "together", keysUrl: "https://api.together.ai/settings/api-keys" }, { id: "fireworks", label: "Fireworks AI", baseUrl: "https://api.fireworks.ai/inference/v1", avatar: "fireworks", keysUrl: "https://fireworks.ai/account/api-keys" }, { id: "cerebras", label: "Cerebras", baseUrl: "https://api.cerebras.ai/v1", avatar: "cerebras", keysUrl: "https://cloud.cerebras.ai" }, { id: "ollama", label: "Ollama", baseUrl: "http://localhost:11434/v1", avatar: "ollama", noKey: true, hint: "Runs on your computer. Start it with OLLAMA_ORIGINS set to this page's address; Chrome may ask to allow access to local devices." }, { id: "lmstudio", label: "LM Studio", baseUrl: "http://localhost:1234/v1", avatar: "lmstudio", noKey: true, hint: "Runs on your computer. Turn on CORS in LM Studio's server settings; Chrome may ask to allow access to local devices." }, { id: "custom", label: "Other", baseUrl: "", hint: "Any OpenAI-compatible Chat Completions API." }, ]; export const findPreset = (id) => API_PRESETS.find((p) => p.id === id) ?? API_PRESETS[API_PRESETS.length - 1]; // Keys live apart from the settings, one per saved API model: in localStorage when the // user asked to remember it, otherwise in sessionStorage (gone when the tab closes). const keyName = (id) => `lwa-api-key:${id}`; const storages = () => { try { return [localStorage, sessionStorage]; } catch { return []; } }; export function loadApiKey(id) { for (const st of storages()) { const v = st.getItem(keyName(id)); if (v != null) return v; } return ""; } export function saveApiKey(id, key, remember) { deleteApiKey(id); try { if (key) (remember ? localStorage : sessionStorage).setItem(keyName(id), key); } catch {} } export function deleteApiKey(id) { for (const st of storages()) st.removeItem(keyName(id)); } export const apiKeyRemembered = (id) => { try { return localStorage.getItem(keyName(id)) != null; } catch { return false; } }; /** Moves the single key of v0.4 (before multiple API models) to the given model id. */ export function migrateLegacyKey(id) { for (const st of storages()) { const v = st.getItem("lwa-api-key"); if (v != null) { st.setItem(keyName(id), v); st.removeItem("lwa-api-key"); } } } const endpoint = (baseUrl) => baseUrl.replace(/\/+$/, "").replace(/\/chat\/completions$/, "") + "/chat/completions"; function headersFor(key, extra) { const h = { "Content-Type": "application/json", ...extra }; if (key) h.Authorization = `Bearer ${key}`; return h; } async function httpError(res) { let detail = ""; try { const body = await res.text(); try { const j = JSON.parse(body); detail = j.error?.message ?? j.message ?? j.error ?? body; } catch { detail = body; } } catch {} detail = String(detail).slice(0, 300); const hint = res.status === 401 || res.status === 403 ? "The provider rejected the API key." : res.status === 404 ? "Endpoint or model not found. Check the base URL and model ID." : res.status === 429 ? "Rate limited or out of credits." : res.status === 400 ? "The provider rejected the request (does this model support tool calling?)." : `The provider returned HTTP ${res.status}.`; return new Error(detail ? `${hint} ${detail}` : hint); } function networkError(baseUrl, err) { let host = baseUrl; try { host = new URL(baseUrl).host; } catch {} return new Error(`Couldn't reach ${host}. Check the URL and your internet connection. The provider must also allow requests from web pages (CORS). (${err.message})`); } // ---------- history translation ---------- const argValue = (v) => (typeof v === "string" ? v : JSON.stringify(v)); /** One API tool call rendered in the LFM3 format that toolcalls.js parses. */ function renderCall(name, rawArgs, final) { let args = null; try { args = rawArgs.trim() ? JSON.parse(rawArgs) : {}; } catch {} if (args && typeof args === "object" && !Array.isArray(args)) { const params = Object.entries(args).map(([k, v]) => `\n${argValue(v)}\n\n`).join(""); return `\n\n${params}\n`; } // Arguments still streaming (incomplete JSON): show the call as being prepared. if (!final) return `\n\n`; // Invalid JSON at the end: run it with no arguments so the agent reports the problem. return `\n\n\n`; } /** Agent history (LFM3 text format) -> Chat Completions messages with native tool calls. */ export function toApiMessages(messages) { const out = []; let pending = []; // tool_call ids still waiting for a result let n = 0; const flush = () => { for (const id of pending) out.push({ role: "tool", tool_call_id: id, content: "(not run)" }); pending = []; }; for (const m of messages) { if (m.role === "assistant") { flush(); const p = parseAssistant(m.content, { thinkingOpen: false }); const calls = p.calls.map((c) => ({ id: `call_${++n}`, type: "function", function: { name: c.name, arguments: JSON.stringify(c.arguments) }, })); const msg = { role: "assistant", content: p.content || (calls.length ? null : "(no reply)") }; if (calls.length) msg.tool_calls = calls; out.push(msg); pending = calls.map((c) => c.id); } else if (m.role === "tool") { const id = pending.shift(); out.push(id ? { role: "tool", tool_call_id: id, content: m.content } : { role: "user", content: `Tool result:\n${m.content}` }); } else { flush(); out.push({ role: m.role, content: m.content }); } } flush(); return out; } // ---------- backend ---------- export class RemoteLLM { /** config: { baseUrl, model, key, headers, getKey? } (getKey: async, returns a fresh key, e.g. a refreshed sign-in token) */ constructor(config) { this.config = config; this.ready = false; this.remote = true; this.onProgress = null; this.onStatus = null; this.onGpu = null; this.ctrl = null; } get label() { return this.config.model; } async load() { if (!this.config.baseUrl || !this.config.model) throw new Error("Set the API base URL and model first."); this.ready = true; } stop() { this.ctrl?.abort(); } resetCache() {} dispose() { this.stop(); } /** Sends a tiny request to check the URL, key and model. Resolves with the reply text. */ static async test(config) { let res; try { res = await fetch(endpoint(config.baseUrl), { method: "POST", headers: headersFor(config.key, config.headers), body: JSON.stringify({ model: config.model, messages: [{ role: "user", content: "Reply with the word OK." }] }), }); } catch (err) { throw networkError(config.baseUrl, err); } if (!res.ok) throw await httpError(res); const j = await res.json().catch(() => null); const msg = j?.choices?.[0]?.message; if (!msg) throw new Error("The reply wasn't in the OpenAI Chat Completions format."); return String(msg.content ?? "").trim() || "(empty reply)"; } /** The provider's model IDs (GET /models), sorted; for the model ID field's suggestions. */ static async listModels(config) { const url = config.baseUrl.replace(/\/+$/, "").replace(/\/chat\/completions$/, "") + "/models"; let res; try { const h = headersFor(config.key, config.headers); delete h["Content-Type"]; res = await fetch(url, { headers: h }); } catch (err) { throw networkError(config.baseUrl, err); } if (!res.ok) throw await httpError(res); const j = await res.json().catch(() => null); let list = Array.isArray(j) ? j : j?.data ?? j?.models ?? []; // The Hugging Face router says which models support tool calling; the agent needs it. if (list.some((m) => Array.isArray(m?.providers) && m.providers.some((p) => "supports_tools" in p))) { list = list.filter((m) => m.providers?.some((p) => p.supports_tools)); } return [...new Set(list.map((m) => (typeof m === "string" ? m : m.id ?? m.name)).filter(Boolean))].sort(); } async generate(messages, tools, options = {}, onEvent = null) { const { baseUrl, model, headers } = this.config; const key = this.config.getKey ? await this.config.getKey() : this.config.key; const ctrl = new AbortController(); this.ctrl = ctrl; const t0 = performance.now(); let tFirst = 0; let reasoning = ""; let content = ""; const calls = []; // { name, args } by stream index let finish = null; let usage = null; let stopped = false; const render = (final = false) => { let r = reasoning; let c = content; // Some models put their reasoning inside tags in the content. const m = c.match(/^\s*([\s\S]*?)(<\/think>|$)/); if (m) { r += m[1]; c = c.slice(m[0].length); } let s = r + "" + c; for (const call of calls) if (call?.name) s += renderCall(call.name, call.args, final); return s; }; const absorb = (delta) => { if (!tFirst) tFirst = performance.now(); reasoning += delta.reasoning_content ?? delta.reasoning ?? ""; content += delta.content ?? ""; for (const tc of delta.tool_calls ?? []) { const i = tc.index ?? calls.length; calls[i] ??= { name: "", args: "" }; if (tc.function?.name) calls[i].name += tc.function.name; if (tc.function?.arguments) calls[i].args += typeof tc.function.arguments === "string" ? tc.function.arguments : JSON.stringify(tc.function.arguments); } }; try { let res; try { res = await fetch(endpoint(baseUrl), { method: "POST", headers: headersFor(key, headers), body: JSON.stringify({ model, messages: toApiMessages(messages), tools, tool_choice: "auto", stream: true }), signal: ctrl.signal, }); } catch (err) { if (err.name === "AbortError") throw err; throw networkError(baseUrl, err); } if (!res.ok) throw await httpError(res); if (!(res.headers.get("Content-Type") ?? "").includes("text/event-stream")) { // Provider ignored stream:true and sent one JSON reply. const j = await res.json(); if (j.error) throw new Error(j.error.message ?? JSON.stringify(j.error)); const ch = j.choices?.[0]; if (ch?.message) absorb(ch.message); finish = ch?.finish_reason ?? null; usage = j.usage ?? null; } else { const reader = res.body.pipeThrough(new TextDecoderStream()).getReader(); let buf = ""; for (;;) { const { done, value } = await reader.read(); if (done) break; buf += value; let nl; while ((nl = buf.indexOf("\n")) >= 0) { const line = buf.slice(0, nl).trim(); buf = buf.slice(nl + 1); if (!line.startsWith("data:")) continue; const data = line.slice(5).trim(); if (!data || data === "[DONE]") continue; let j; try { j = JSON.parse(data); } catch { continue; } if (j.error) throw new Error(j.error.message ?? JSON.stringify(j.error)); if (j.usage) usage = j.usage; const ch = j.choices?.[0]; if (!ch) continue; if (ch.delta) absorb(ch.delta); if (ch.finish_reason) finish = ch.finish_reason; onEvent?.({ type: "replace", text: render() }); } } } } catch (err) { if (err.name !== "AbortError") throw err; stopped = true; } finally { if (this.ctrl === ctrl) this.ctrl = null; } const text = render(true); onEvent?.({ type: "replace", text }); const end = performance.now(); const promptChars = messages.reduce((n, m) => n + (m.content?.length ?? 0), 0); return { text, stopped, hitLimit: finish === "length", thinkingCut: null, promptTokens: usage?.prompt_tokens ?? Math.ceil(promptChars / 3.5), genTokens: usage?.completion_tokens ?? Math.ceil((reasoning.length + content.length) / 3.5), reused: 0, newPrefill: 0, prefillMs: tFirst ? tFirst - t0 : end - t0, genMs: tFirst ? end - tFirst : 0, }; } }