LocalWebAgent / js /remote-llm.js
ereniko's picture
v0.8.0: Sign in with Hugging Face for API models
46d9e41 unverified
Raw History Blame Contribute Delete
14.9 kB
// Remote model backend: any OpenAI-compatible Chat Completions API (OpenAI, OpenRouter,
// the Hugging Face router, Groq, a local Ollama/LM Studio server, ...). Used when the
// local model can't run on this device. The Linux machine still runs in the tab.
//
// It has the same interface as LLM (llm.js), so the agent loop is unchanged: the agent's
// history (LFM3 text format) is translated into API messages with native tool calls, and
// the streamed reply is rendered back into that text format for the agent to parse.
import { parseAssistant } from "./toolcalls.js";
// Providers with an OpenAI-compatible Chat Completions API that accept requests from web
// pages (CORS preflight checked for each). `avatar` is the provider's Hugging Face avatar
// in vendor/avatars/ (none for "Other").
export const API_PRESETS = [
{ id: "huggingface", signIn: true, defaultModel: "openai/gpt-oss-120b", label: "Hugging Face", baseUrl: "https://router.huggingface.co/v1", avatar: "huggingface", keysUrl: "https://huggingface.co/settings/tokens",
hint: "Open models from many inference providers, billed to your Hugging Face account." },
{ id: "openrouter", label: "OpenRouter", baseUrl: "https://openrouter.ai/api/v1", avatar: "openrouter", keysUrl: "https://openrouter.ai/keys",
hint: "Hundreds of models from every major lab behind one key." },
{ id: "openai", label: "OpenAI", baseUrl: "https://api.openai.com/v1", avatar: "openai", keysUrl: "https://platform.openai.com/api-keys" },
{ id: "anthropic", label: "Anthropic", baseUrl: "https://api.anthropic.com/v1", avatar: "anthropic", keysUrl: "https://console.anthropic.com/settings/keys",
headers: { "anthropic-dangerous-direct-browser-access": "true" }, hint: "Uses Anthropic's OpenAI-compatible endpoint." },
{ id: "gemini", label: "Google Gemini", baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai", avatar: "google", keysUrl: "https://aistudio.google.com/apikey" },
{ id: "groq", label: "Groq", baseUrl: "https://api.groq.com/openai/v1", avatar: "groq", keysUrl: "https://console.groq.com/keys" },
{ id: "mistral", label: "Mistral AI", baseUrl: "https://api.mistral.ai/v1", avatar: "mistral", keysUrl: "https://console.mistral.ai/api-keys" },
{ id: "deepseek", label: "DeepSeek", baseUrl: "https://api.deepseek.com/v1", avatar: "deepseek", keysUrl: "https://platform.deepseek.com/api_keys" },
{ id: "xai", label: "xAI", baseUrl: "https://api.x.ai/v1", avatar: "xai", keysUrl: "https://console.x.ai" },
{ id: "together", label: "Together AI", baseUrl: "https://api.together.xyz/v1", avatar: "together", keysUrl: "https://api.together.ai/settings/api-keys" },
{ id: "fireworks", label: "Fireworks AI", baseUrl: "https://api.fireworks.ai/inference/v1", avatar: "fireworks", keysUrl: "https://fireworks.ai/account/api-keys" },
{ id: "cerebras", label: "Cerebras", baseUrl: "https://api.cerebras.ai/v1", avatar: "cerebras", keysUrl: "https://cloud.cerebras.ai" },
{ id: "ollama", label: "Ollama", baseUrl: "http://localhost:11434/v1", avatar: "ollama", noKey: true,
hint: "Runs on your computer. Start it with OLLAMA_ORIGINS set to this page's address; Chrome may ask to allow access to local devices." },
{ id: "lmstudio", label: "LM Studio", baseUrl: "http://localhost:1234/v1", avatar: "lmstudio", noKey: true,
hint: "Runs on your computer. Turn on CORS in LM Studio's server settings; Chrome may ask to allow access to local devices." },
{ id: "custom", label: "Other", baseUrl: "", hint: "Any OpenAI-compatible Chat Completions API." },
];
export const findPreset = (id) => API_PRESETS.find((p) => p.id === id) ?? API_PRESETS[API_PRESETS.length - 1];
// Keys live apart from the settings, one per saved API model: in localStorage when the
// user asked to remember it, otherwise in sessionStorage (gone when the tab closes).
const keyName = (id) => `lwa-api-key:${id}`;
const storages = () => { try { return [localStorage, sessionStorage]; } catch { return []; } };
export function loadApiKey(id) {
for (const st of storages()) { const v = st.getItem(keyName(id)); if (v != null) return v; }
return "";
}
export function saveApiKey(id, key, remember) {
deleteApiKey(id);
try { if (key) (remember ? localStorage : sessionStorage).setItem(keyName(id), key); } catch {}
}
export function deleteApiKey(id) {
for (const st of storages()) st.removeItem(keyName(id));
}
export const apiKeyRemembered = (id) => { try { return localStorage.getItem(keyName(id)) != null; } catch { return false; } };
/** Moves the single key of v0.4 (before multiple API models) to the given model id. */
export function migrateLegacyKey(id) {
for (const st of storages()) {
const v = st.getItem("lwa-api-key");
if (v != null) { st.setItem(keyName(id), v); st.removeItem("lwa-api-key"); }
}
}
const endpoint = (baseUrl) => baseUrl.replace(/\/+$/, "").replace(/\/chat\/completions$/, "") + "/chat/completions";
function headersFor(key, extra) {
const h = { "Content-Type": "application/json", ...extra };
if (key) h.Authorization = `Bearer ${key}`;
return h;
}
async function httpError(res) {
let detail = "";
try {
const body = await res.text();
try { const j = JSON.parse(body); detail = j.error?.message ?? j.message ?? j.error ?? body; } catch { detail = body; }
} catch {}
detail = String(detail).slice(0, 300);
const hint = res.status === 401 || res.status === 403 ? "The provider rejected the API key."
: res.status === 404 ? "Endpoint or model not found. Check the base URL and model ID."
: res.status === 429 ? "Rate limited or out of credits."
: res.status === 400 ? "The provider rejected the request (does this model support tool calling?)."
: `The provider returned HTTP ${res.status}.`;
return new Error(detail ? `${hint} ${detail}` : hint);
}
function networkError(baseUrl, err) {
let host = baseUrl;
try { host = new URL(baseUrl).host; } catch {}
return new Error(`Couldn't reach ${host}. Check the URL and your internet connection. The provider must also allow requests from web pages (CORS). (${err.message})`);
}
// ---------- history translation ----------
const argValue = (v) => (typeof v === "string" ? v : JSON.stringify(v));
/** One API tool call rendered in the LFM3 format that toolcalls.js parses. */
function renderCall(name, rawArgs, final) {
let args = null;
try { args = rawArgs.trim() ? JSON.parse(rawArgs) : {}; } catch {}
if (args && typeof args === "object" && !Array.isArray(args)) {
const params = Object.entries(args).map(([k, v]) => `<parameter=${k}>\n${argValue(v)}\n</parameter>\n`).join("");
return `<tool_call>\n<function=${name}>\n${params}</function>\n</tool_call>`;
}
// Arguments still streaming (incomplete JSON): show the call as being prepared.
if (!final) return `<tool_call>\n<function=${name}>\n`;
// Invalid JSON at the end: run it with no arguments so the agent reports the problem.
return `<tool_call>\n<function=${name}>\n</function>\n</tool_call>`;
}
/** Agent history (LFM3 text format) -> Chat Completions messages with native tool calls. */
export function toApiMessages(messages) {
const out = [];
let pending = []; // tool_call ids still waiting for a result
let n = 0;
const flush = () => {
for (const id of pending) out.push({ role: "tool", tool_call_id: id, content: "(not run)" });
pending = [];
};
for (const m of messages) {
if (m.role === "assistant") {
flush();
const p = parseAssistant(m.content, { thinkingOpen: false });
const calls = p.calls.map((c) => ({
id: `call_${++n}`,
type: "function",
function: { name: c.name, arguments: JSON.stringify(c.arguments) },
}));
const msg = { role: "assistant", content: p.content || (calls.length ? null : "(no reply)") };
if (calls.length) msg.tool_calls = calls;
out.push(msg);
pending = calls.map((c) => c.id);
} else if (m.role === "tool") {
const id = pending.shift();
out.push(id ? { role: "tool", tool_call_id: id, content: m.content } : { role: "user", content: `Tool result:\n${m.content}` });
} else {
flush();
out.push({ role: m.role, content: m.content });
}
}
flush();
return out;
}
// ---------- backend ----------
export class RemoteLLM {
/** config: { baseUrl, model, key, headers, getKey? } (getKey: async, returns a fresh key, e.g. a refreshed sign-in token) */
constructor(config) {
this.config = config;
this.ready = false;
this.remote = true;
this.onProgress = null;
this.onStatus = null;
this.onGpu = null;
this.ctrl = null;
}
get label() { return this.config.model; }
async load() {
if (!this.config.baseUrl || !this.config.model) throw new Error("Set the API base URL and model first.");
this.ready = true;
}
stop() { this.ctrl?.abort(); }
resetCache() {}
dispose() { this.stop(); }
/** Sends a tiny request to check the URL, key and model. Resolves with the reply text. */
static async test(config) {
let res;
try {
res = await fetch(endpoint(config.baseUrl), {
method: "POST",
headers: headersFor(config.key, config.headers),
body: JSON.stringify({ model: config.model, messages: [{ role: "user", content: "Reply with the word OK." }] }),
});
} catch (err) { throw networkError(config.baseUrl, err); }
if (!res.ok) throw await httpError(res);
const j = await res.json().catch(() => null);
const msg = j?.choices?.[0]?.message;
if (!msg) throw new Error("The reply wasn't in the OpenAI Chat Completions format.");
return String(msg.content ?? "").trim() || "(empty reply)";
}
/** The provider's model IDs (GET /models), sorted; for the model ID field's suggestions. */
static async listModels(config) {
const url = config.baseUrl.replace(/\/+$/, "").replace(/\/chat\/completions$/, "") + "/models";
let res;
try {
const h = headersFor(config.key, config.headers);
delete h["Content-Type"];
res = await fetch(url, { headers: h });
} catch (err) { throw networkError(config.baseUrl, err); }
if (!res.ok) throw await httpError(res);
const j = await res.json().catch(() => null);
let list = Array.isArray(j) ? j : j?.data ?? j?.models ?? [];
// The Hugging Face router says which models support tool calling; the agent needs it.
if (list.some((m) => Array.isArray(m?.providers) && m.providers.some((p) => "supports_tools" in p))) {
list = list.filter((m) => m.providers?.some((p) => p.supports_tools));
}
return [...new Set(list.map((m) => (typeof m === "string" ? m : m.id ?? m.name)).filter(Boolean))].sort();
}
async generate(messages, tools, options = {}, onEvent = null) {
const { baseUrl, model, headers } = this.config;
const key = this.config.getKey ? await this.config.getKey() : this.config.key;
const ctrl = new AbortController();
this.ctrl = ctrl;
const t0 = performance.now();
let tFirst = 0;
let reasoning = "";
let content = "";
const calls = []; // { name, args } by stream index
let finish = null;
let usage = null;
let stopped = false;
const render = (final = false) => {
let r = reasoning;
let c = content;
// Some models put their reasoning inside <think> tags in the content.
const m = c.match(/^\s*<think>([\s\S]*?)(<\/think>|$)/);
if (m) { r += m[1]; c = c.slice(m[0].length); }
let s = r + "</think>" + c;
for (const call of calls) if (call?.name) s += renderCall(call.name, call.args, final);
return s;
};
const absorb = (delta) => {
if (!tFirst) tFirst = performance.now();
reasoning += delta.reasoning_content ?? delta.reasoning ?? "";
content += delta.content ?? "";
for (const tc of delta.tool_calls ?? []) {
const i = tc.index ?? calls.length;
calls[i] ??= { name: "", args: "" };
if (tc.function?.name) calls[i].name += tc.function.name;
if (tc.function?.arguments) calls[i].args += typeof tc.function.arguments === "string" ? tc.function.arguments : JSON.stringify(tc.function.arguments);
}
};
try {
let res;
try {
res = await fetch(endpoint(baseUrl), {
method: "POST",
headers: headersFor(key, headers),
body: JSON.stringify({ model, messages: toApiMessages(messages), tools, tool_choice: "auto", stream: true }),
signal: ctrl.signal,
});
} catch (err) {
if (err.name === "AbortError") throw err;
throw networkError(baseUrl, err);
}
if (!res.ok) throw await httpError(res);
if (!(res.headers.get("Content-Type") ?? "").includes("text/event-stream")) {
// Provider ignored stream:true and sent one JSON reply.
const j = await res.json();
if (j.error) throw new Error(j.error.message ?? JSON.stringify(j.error));
const ch = j.choices?.[0];
if (ch?.message) absorb(ch.message);
finish = ch?.finish_reason ?? null;
usage = j.usage ?? null;
} else {
const reader = res.body.pipeThrough(new TextDecoderStream()).getReader();
let buf = "";
for (;;) {
const { done, value } = await reader.read();
if (done) break;
buf += value;
let nl;
while ((nl = buf.indexOf("\n")) >= 0) {
const line = buf.slice(0, nl).trim();
buf = buf.slice(nl + 1);
if (!line.startsWith("data:")) continue;
const data = line.slice(5).trim();
if (!data || data === "[DONE]") continue;
let j;
try { j = JSON.parse(data); } catch { continue; }
if (j.error) throw new Error(j.error.message ?? JSON.stringify(j.error));
if (j.usage) usage = j.usage;
const ch = j.choices?.[0];
if (!ch) continue;
if (ch.delta) absorb(ch.delta);
if (ch.finish_reason) finish = ch.finish_reason;
onEvent?.({ type: "replace", text: render() });
}
}
}
} catch (err) {
if (err.name !== "AbortError") throw err;
stopped = true;
} finally {
if (this.ctrl === ctrl) this.ctrl = null;
}
const text = render(true);
onEvent?.({ type: "replace", text });
const end = performance.now();
const promptChars = messages.reduce((n, m) => n + (m.content?.length ?? 0), 0);
return {
text,
stopped,
hitLimit: finish === "length",
thinkingCut: null,
promptTokens: usage?.prompt_tokens ?? Math.ceil(promptChars / 3.5),
genTokens: usage?.completion_tokens ?? Math.ceil((reasoning.length + content.length) / 3.5),
reused: 0,
newPrefill: 0,
prefillMs: tFirst ? tFirst - t0 : end - t0,
genMs: tFirst ? end - tFirst : 0,
};
}
}