Spaces:
Running
Running
Download js/remote-llm.js from ereniko/LocalWebAgent: direct link, hf CLI and curl.
- Browser
- Download file 14.9 kB
-
https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/js/remote-llm.js
- Command line
-
hf download hf://spaces/ereniko/LocalWebAgent/js/remote-llm.js
-
curl -L -o remote-llm.js https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/js/remote-llm.js
14.9 kB
| // Remote model backend: any OpenAI-compatible Chat Completions API (OpenAI, OpenRouter, | |
| // the Hugging Face router, Groq, a local Ollama/LM Studio server, ...). Used when the | |
| // local model can't run on this device. The Linux machine still runs in the tab. | |
| // | |
| // It has the same interface as LLM (llm.js), so the agent loop is unchanged: the agent's | |
| // history (LFM3 text format) is translated into API messages with native tool calls, and | |
| // the streamed reply is rendered back into that text format for the agent to parse. | |
| import { parseAssistant } from "./toolcalls.js"; | |
| // Providers with an OpenAI-compatible Chat Completions API that accept requests from web | |
| // pages (CORS preflight checked for each). `avatar` is the provider's Hugging Face avatar | |
| // in vendor/avatars/ (none for "Other"). | |
| export const API_PRESETS = [ | |
| { id: "huggingface", signIn: true, defaultModel: "openai/gpt-oss-120b", label: "Hugging Face", baseUrl: "https://router.huggingface.co/v1", avatar: "huggingface", keysUrl: "https://huggingface.co/settings/tokens", | |
| hint: "Open models from many inference providers, billed to your Hugging Face account." }, | |
| { id: "openrouter", label: "OpenRouter", baseUrl: "https://openrouter.ai/api/v1", avatar: "openrouter", keysUrl: "https://openrouter.ai/keys", | |
| hint: "Hundreds of models from every major lab behind one key." }, | |
| { id: "openai", label: "OpenAI", baseUrl: "https://api.openai.com/v1", avatar: "openai", keysUrl: "https://platform.openai.com/api-keys" }, | |
| { id: "anthropic", label: "Anthropic", baseUrl: "https://api.anthropic.com/v1", avatar: "anthropic", keysUrl: "https://console.anthropic.com/settings/keys", | |
| headers: { "anthropic-dangerous-direct-browser-access": "true" }, hint: "Uses Anthropic's OpenAI-compatible endpoint." }, | |
| { id: "gemini", label: "Google Gemini", baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai", avatar: "google", keysUrl: "https://aistudio.google.com/apikey" }, | |
| { id: "groq", label: "Groq", baseUrl: "https://api.groq.com/openai/v1", avatar: "groq", keysUrl: "https://console.groq.com/keys" }, | |
| { id: "mistral", label: "Mistral AI", baseUrl: "https://api.mistral.ai/v1", avatar: "mistral", keysUrl: "https://console.mistral.ai/api-keys" }, | |
| { id: "deepseek", label: "DeepSeek", baseUrl: "https://api.deepseek.com/v1", avatar: "deepseek", keysUrl: "https://platform.deepseek.com/api_keys" }, | |
| { id: "xai", label: "xAI", baseUrl: "https://api.x.ai/v1", avatar: "xai", keysUrl: "https://console.x.ai" }, | |
| { id: "together", label: "Together AI", baseUrl: "https://api.together.xyz/v1", avatar: "together", keysUrl: "https://api.together.ai/settings/api-keys" }, | |
| { id: "fireworks", label: "Fireworks AI", baseUrl: "https://api.fireworks.ai/inference/v1", avatar: "fireworks", keysUrl: "https://fireworks.ai/account/api-keys" }, | |
| { id: "cerebras", label: "Cerebras", baseUrl: "https://api.cerebras.ai/v1", avatar: "cerebras", keysUrl: "https://cloud.cerebras.ai" }, | |
| { id: "ollama", label: "Ollama", baseUrl: "http://localhost:11434/v1", avatar: "ollama", noKey: true, | |
| hint: "Runs on your computer. Start it with OLLAMA_ORIGINS set to this page's address; Chrome may ask to allow access to local devices." }, | |
| { id: "lmstudio", label: "LM Studio", baseUrl: "http://localhost:1234/v1", avatar: "lmstudio", noKey: true, | |
| hint: "Runs on your computer. Turn on CORS in LM Studio's server settings; Chrome may ask to allow access to local devices." }, | |
| { id: "custom", label: "Other", baseUrl: "", hint: "Any OpenAI-compatible Chat Completions API." }, | |
| ]; | |
| export const findPreset = (id) => API_PRESETS.find((p) => p.id === id) ?? API_PRESETS[API_PRESETS.length - 1]; | |
| // Keys live apart from the settings, one per saved API model: in localStorage when the | |
| // user asked to remember it, otherwise in sessionStorage (gone when the tab closes). | |
| const keyName = (id) => `lwa-api-key:${id}`; | |
| const storages = () => { try { return [localStorage, sessionStorage]; } catch { return []; } }; | |
| export function loadApiKey(id) { | |
| for (const st of storages()) { const v = st.getItem(keyName(id)); if (v != null) return v; } | |
| return ""; | |
| } | |
| export function saveApiKey(id, key, remember) { | |
| deleteApiKey(id); | |
| try { if (key) (remember ? localStorage : sessionStorage).setItem(keyName(id), key); } catch {} | |
| } | |
| export function deleteApiKey(id) { | |
| for (const st of storages()) st.removeItem(keyName(id)); | |
| } | |
| export const apiKeyRemembered = (id) => { try { return localStorage.getItem(keyName(id)) != null; } catch { return false; } }; | |
| /** Moves the single key of v0.4 (before multiple API models) to the given model id. */ | |
| export function migrateLegacyKey(id) { | |
| for (const st of storages()) { | |
| const v = st.getItem("lwa-api-key"); | |
| if (v != null) { st.setItem(keyName(id), v); st.removeItem("lwa-api-key"); } | |
| } | |
| } | |
| const endpoint = (baseUrl) => baseUrl.replace(/\/+$/, "").replace(/\/chat\/completions$/, "") + "/chat/completions"; | |
| function headersFor(key, extra) { | |
| const h = { "Content-Type": "application/json", ...extra }; | |
| if (key) h.Authorization = `Bearer ${key}`; | |
| return h; | |
| } | |
| async function httpError(res) { | |
| let detail = ""; | |
| try { | |
| const body = await res.text(); | |
| try { const j = JSON.parse(body); detail = j.error?.message ?? j.message ?? j.error ?? body; } catch { detail = body; } | |
| } catch {} | |
| detail = String(detail).slice(0, 300); | |
| const hint = res.status === 401 || res.status === 403 ? "The provider rejected the API key." | |
| : res.status === 404 ? "Endpoint or model not found. Check the base URL and model ID." | |
| : res.status === 429 ? "Rate limited or out of credits." | |
| : res.status === 400 ? "The provider rejected the request (does this model support tool calling?)." | |
| : `The provider returned HTTP ${res.status}.`; | |
| return new Error(detail ? `${hint} ${detail}` : hint); | |
| } | |
| function networkError(baseUrl, err) { | |
| let host = baseUrl; | |
| try { host = new URL(baseUrl).host; } catch {} | |
| return new Error(`Couldn't reach ${host}. Check the URL and your internet connection. The provider must also allow requests from web pages (CORS). (${err.message})`); | |
| } | |
| // ---------- history translation ---------- | |
| const argValue = (v) => (typeof v === "string" ? v : JSON.stringify(v)); | |
| /** One API tool call rendered in the LFM3 format that toolcalls.js parses. */ | |
| function renderCall(name, rawArgs, final) { | |
| let args = null; | |
| try { args = rawArgs.trim() ? JSON.parse(rawArgs) : {}; } catch {} | |
| if (args && typeof args === "object" && !Array.isArray(args)) { | |
| const params = Object.entries(args).map(([k, v]) => `<parameter=${k}>\n${argValue(v)}\n</parameter>\n`).join(""); | |
| return `<tool_call>\n<function=${name}>\n${params}</function>\n</tool_call>`; | |
| } | |
| // Arguments still streaming (incomplete JSON): show the call as being prepared. | |
| if (!final) return `<tool_call>\n<function=${name}>\n`; | |
| // Invalid JSON at the end: run it with no arguments so the agent reports the problem. | |
| return `<tool_call>\n<function=${name}>\n</function>\n</tool_call>`; | |
| } | |
| /** Agent history (LFM3 text format) -> Chat Completions messages with native tool calls. */ | |
| export function toApiMessages(messages) { | |
| const out = []; | |
| let pending = []; // tool_call ids still waiting for a result | |
| let n = 0; | |
| const flush = () => { | |
| for (const id of pending) out.push({ role: "tool", tool_call_id: id, content: "(not run)" }); | |
| pending = []; | |
| }; | |
| for (const m of messages) { | |
| if (m.role === "assistant") { | |
| flush(); | |
| const p = parseAssistant(m.content, { thinkingOpen: false }); | |
| const calls = p.calls.map((c) => ({ | |
| id: `call_${++n}`, | |
| type: "function", | |
| function: { name: c.name, arguments: JSON.stringify(c.arguments) }, | |
| })); | |
| const msg = { role: "assistant", content: p.content || (calls.length ? null : "(no reply)") }; | |
| if (calls.length) msg.tool_calls = calls; | |
| out.push(msg); | |
| pending = calls.map((c) => c.id); | |
| } else if (m.role === "tool") { | |
| const id = pending.shift(); | |
| out.push(id ? { role: "tool", tool_call_id: id, content: m.content } : { role: "user", content: `Tool result:\n${m.content}` }); | |
| } else { | |
| flush(); | |
| out.push({ role: m.role, content: m.content }); | |
| } | |
| } | |
| flush(); | |
| return out; | |
| } | |
| // ---------- backend ---------- | |
| export class RemoteLLM { | |
| /** config: { baseUrl, model, key, headers, getKey? } (getKey: async, returns a fresh key, e.g. a refreshed sign-in token) */ | |
| constructor(config) { | |
| this.config = config; | |
| this.ready = false; | |
| this.remote = true; | |
| this.onProgress = null; | |
| this.onStatus = null; | |
| this.onGpu = null; | |
| this.ctrl = null; | |
| } | |
| get label() { return this.config.model; } | |
| async load() { | |
| if (!this.config.baseUrl || !this.config.model) throw new Error("Set the API base URL and model first."); | |
| this.ready = true; | |
| } | |
| stop() { this.ctrl?.abort(); } | |
| resetCache() {} | |
| dispose() { this.stop(); } | |
| /** Sends a tiny request to check the URL, key and model. Resolves with the reply text. */ | |
| static async test(config) { | |
| let res; | |
| try { | |
| res = await fetch(endpoint(config.baseUrl), { | |
| method: "POST", | |
| headers: headersFor(config.key, config.headers), | |
| body: JSON.stringify({ model: config.model, messages: [{ role: "user", content: "Reply with the word OK." }] }), | |
| }); | |
| } catch (err) { throw networkError(config.baseUrl, err); } | |
| if (!res.ok) throw await httpError(res); | |
| const j = await res.json().catch(() => null); | |
| const msg = j?.choices?.[0]?.message; | |
| if (!msg) throw new Error("The reply wasn't in the OpenAI Chat Completions format."); | |
| return String(msg.content ?? "").trim() || "(empty reply)"; | |
| } | |
| /** The provider's model IDs (GET /models), sorted; for the model ID field's suggestions. */ | |
| static async listModels(config) { | |
| const url = config.baseUrl.replace(/\/+$/, "").replace(/\/chat\/completions$/, "") + "/models"; | |
| let res; | |
| try { | |
| const h = headersFor(config.key, config.headers); | |
| delete h["Content-Type"]; | |
| res = await fetch(url, { headers: h }); | |
| } catch (err) { throw networkError(config.baseUrl, err); } | |
| if (!res.ok) throw await httpError(res); | |
| const j = await res.json().catch(() => null); | |
| let list = Array.isArray(j) ? j : j?.data ?? j?.models ?? []; | |
| // The Hugging Face router says which models support tool calling; the agent needs it. | |
| if (list.some((m) => Array.isArray(m?.providers) && m.providers.some((p) => "supports_tools" in p))) { | |
| list = list.filter((m) => m.providers?.some((p) => p.supports_tools)); | |
| } | |
| return [...new Set(list.map((m) => (typeof m === "string" ? m : m.id ?? m.name)).filter(Boolean))].sort(); | |
| } | |
| async generate(messages, tools, options = {}, onEvent = null) { | |
| const { baseUrl, model, headers } = this.config; | |
| const key = this.config.getKey ? await this.config.getKey() : this.config.key; | |
| const ctrl = new AbortController(); | |
| this.ctrl = ctrl; | |
| const t0 = performance.now(); | |
| let tFirst = 0; | |
| let reasoning = ""; | |
| let content = ""; | |
| const calls = []; // { name, args } by stream index | |
| let finish = null; | |
| let usage = null; | |
| let stopped = false; | |
| const render = (final = false) => { | |
| let r = reasoning; | |
| let c = content; | |
| // Some models put their reasoning inside <think> tags in the content. | |
| const m = c.match(/^\s*<think>([\s\S]*?)(<\/think>|$)/); | |
| if (m) { r += m[1]; c = c.slice(m[0].length); } | |
| let s = r + "</think>" + c; | |
| for (const call of calls) if (call?.name) s += renderCall(call.name, call.args, final); | |
| return s; | |
| }; | |
| const absorb = (delta) => { | |
| if (!tFirst) tFirst = performance.now(); | |
| reasoning += delta.reasoning_content ?? delta.reasoning ?? ""; | |
| content += delta.content ?? ""; | |
| for (const tc of delta.tool_calls ?? []) { | |
| const i = tc.index ?? calls.length; | |
| calls[i] ??= { name: "", args: "" }; | |
| if (tc.function?.name) calls[i].name += tc.function.name; | |
| if (tc.function?.arguments) calls[i].args += typeof tc.function.arguments === "string" ? tc.function.arguments : JSON.stringify(tc.function.arguments); | |
| } | |
| }; | |
| try { | |
| let res; | |
| try { | |
| res = await fetch(endpoint(baseUrl), { | |
| method: "POST", | |
| headers: headersFor(key, headers), | |
| body: JSON.stringify({ model, messages: toApiMessages(messages), tools, tool_choice: "auto", stream: true }), | |
| signal: ctrl.signal, | |
| }); | |
| } catch (err) { | |
| if (err.name === "AbortError") throw err; | |
| throw networkError(baseUrl, err); | |
| } | |
| if (!res.ok) throw await httpError(res); | |
| if (!(res.headers.get("Content-Type") ?? "").includes("text/event-stream")) { | |
| // Provider ignored stream:true and sent one JSON reply. | |
| const j = await res.json(); | |
| if (j.error) throw new Error(j.error.message ?? JSON.stringify(j.error)); | |
| const ch = j.choices?.[0]; | |
| if (ch?.message) absorb(ch.message); | |
| finish = ch?.finish_reason ?? null; | |
| usage = j.usage ?? null; | |
| } else { | |
| const reader = res.body.pipeThrough(new TextDecoderStream()).getReader(); | |
| let buf = ""; | |
| for (;;) { | |
| const { done, value } = await reader.read(); | |
| if (done) break; | |
| buf += value; | |
| let nl; | |
| while ((nl = buf.indexOf("\n")) >= 0) { | |
| const line = buf.slice(0, nl).trim(); | |
| buf = buf.slice(nl + 1); | |
| if (!line.startsWith("data:")) continue; | |
| const data = line.slice(5).trim(); | |
| if (!data || data === "[DONE]") continue; | |
| let j; | |
| try { j = JSON.parse(data); } catch { continue; } | |
| if (j.error) throw new Error(j.error.message ?? JSON.stringify(j.error)); | |
| if (j.usage) usage = j.usage; | |
| const ch = j.choices?.[0]; | |
| if (!ch) continue; | |
| if (ch.delta) absorb(ch.delta); | |
| if (ch.finish_reason) finish = ch.finish_reason; | |
| onEvent?.({ type: "replace", text: render() }); | |
| } | |
| } | |
| } | |
| } catch (err) { | |
| if (err.name !== "AbortError") throw err; | |
| stopped = true; | |
| } finally { | |
| if (this.ctrl === ctrl) this.ctrl = null; | |
| } | |
| const text = render(true); | |
| onEvent?.({ type: "replace", text }); | |
| const end = performance.now(); | |
| const promptChars = messages.reduce((n, m) => n + (m.content?.length ?? 0), 0); | |
| return { | |
| text, | |
| stopped, | |
| hitLimit: finish === "length", | |
| thinkingCut: null, | |
| promptTokens: usage?.prompt_tokens ?? Math.ceil(promptChars / 3.5), | |
| genTokens: usage?.completion_tokens ?? Math.ceil((reasoning.length + content.length) / 3.5), | |
| reused: 0, | |
| newPrefill: 0, | |
| prefillMs: tFirst ? tFirst - t0 : end - t0, | |
| genMs: tFirst ? end - tFirst : 0, | |
| }; | |
| } | |
| } | |