Spaces:
Running
Running
Download js/llm-engine.js from ereniko/LocalWebAgent: direct link, hf CLI and curl.
- Browser
- Download file 12.5 kB
-
https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/js/llm-engine.js
- Command line
-
hf download hf://spaces/ereniko/LocalWebAgent/js/llm-engine.js
-
curl -L -o llm-engine.js https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/js/llm-engine.js
12.5 kB
| // LFM3 generation engine, independent of where it runs. The browser worker passes | |
| // in the transformers.js web build; tools/agent-e2e.mjs passes the Node build so the | |
| // exact same logic can be tested off-GPU. | |
| // | |
| // Generation = chunked prefill (LiquidAI: "on WebGPU ... prefill in chunks of at most | |
| // 256 tokens") followed by transformers.js generate(). The KV/conv cache is kept | |
| // between calls: if the next prompt starts with exactly the tokens already in the | |
| // cache (the usual case inside one agent run), only the new tokens are prefilled. | |
| const PREFILL_CHUNK = 256; | |
| const LOOP_WINDOW = 40; // a verbatim repeat of this many tokens while thinking = a loop | |
| const LOOP_MIN_TOKENS = 160; | |
| const THINK_CLOSE = { | |
| loop: "\n\nI am going in circles. I'll stop deliberating and take the next concrete step.\n</think>\n", | |
| budget: "\n\nI have thought about this enough. Time to act.\n</think>\n", | |
| }; | |
| // Maps a graph output (present_*) to the matching cache input, for every cache kind the | |
| // supported architectures use: attention KV, conv (LFM), SSM (Granite), recurrent (Qwen3.5). | |
| // Mirrors transformers.js' own mapping. | |
| const pastName = (k) => k | |
| .replace("present_ssm", "past_ssm").replace("present_conv", "past_conv") | |
| .replace("present_recurrent", "past_recurrent").replace("present", "past_key_values"); | |
| export function createEngine(tf) { | |
| const { AutoTokenizer, AutoModelForCausalLM, DynamicCache, Tensor, TextStreamer, InterruptableStoppingCriteria, StoppingCriteria } = tf; | |
| let tokenizer = null; | |
| let model = null; | |
| let cached = null; // { tokens: number[], cache: DynamicCache } | |
| const stopper = new InterruptableStoppingCriteria(); | |
| let stopRequested = false; | |
| let thinkEndId = -1; | |
| // Stops generation while the model is still inside <think> if it starts repeating | |
| // itself verbatim or exceeds the thinking budget. Inactive once </think> appears. | |
| class ThinkingGuard extends StoppingCriteria { | |
| constructor(promptLen, budget) { | |
| super(); | |
| this.promptLen = promptLen; | |
| this.budget = budget; | |
| this.scanned = promptLen; | |
| this.closed = false; | |
| this.reason = null; | |
| this.cutAt = -1; | |
| } | |
| _call(input_ids) { | |
| if (this.closed) return [false]; | |
| if (this.reason) return [true]; | |
| const ids = input_ids[0]; | |
| for (let i = this.scanned; i < ids.length; i++) { | |
| if (Number(ids[i]) === thinkEndId) { this.closed = true; return [false]; } | |
| } | |
| this.scanned = ids.length; | |
| const len = ids.length - this.promptLen; | |
| if (len >= this.budget) { this.reason = "budget"; this.cutAt = len; return [true]; } | |
| if (len >= LOOP_MIN_TOKENS && len % 8 === 0) { | |
| const g = ids.slice(this.promptLen).map(Number); | |
| const tailStart = g.length - LOOP_WINDOW; | |
| outer: for (let s = 0; s + LOOP_WINDOW <= tailStart; s++) { | |
| for (let k = 0; k < LOOP_WINDOW; k++) if (g[s + k] !== g[tailStart + k]) continue outer; | |
| this.reason = "loop"; | |
| this.cutAt = tailStart; // keep one copy of the repeated passage | |
| return [true]; | |
| } | |
| } | |
| return [false]; | |
| } | |
| } | |
| const int64 = (arr, dims) => new Tensor("int64", BigInt64Array.from(arr, BigInt), dims); | |
| const ones = (n) => new Tensor("int64", new BigInt64Array(n).fill(1n), [1, n]); | |
| const emptyTensor = (type, dims) => new Tensor(type, type === "int64" ? new BigInt64Array(0) : new Float32Array(0), dims); | |
| // modelOptions: extra from_pretrained options (the browser worker uses it to hand | |
| // onnxruntime the weights as a lazily read, disk-backed buffer). | |
| // chatTemplate: overrides the tokenizer's template. emptyInputs: graph inputs to feed as | |
| // empty tensors on every forward pass ({name: [type, dims]}), e.g. the image/audio inputs | |
| // of a multimodal decoder used for text only. | |
| // fullPrefill: let transformers.js prefill the whole prompt itself and don't reuse the | |
| // cache between calls. Used for vision-language decoders run text-only (Qwen3.5 2B/4B), | |
| // whose multimodal rotary positions only transformers.js' own generate() computes. | |
| let chatTemplate; | |
| let fullPrefill = false; | |
| async function load(modelId, { dtype, device, progress_callback, onStatus, modelOptions, chatTemplate: template, emptyInputs, fullPrefill: full = false } = {}) { | |
| chatTemplate = template; | |
| fullPrefill = full; | |
| tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback }); | |
| onStatus?.("Downloading / loading model weights"); | |
| model = await AutoModelForCausalLM.from_pretrained(modelId, { dtype, device, progress_callback, ...modelOptions }); | |
| const te = tokenizer.encode("</think>", { add_special_tokens: false }); | |
| thinkEndId = te.length === 1 ? te[0] : -1; | |
| // Some graphs (LFM3) take `logits_to_keep` (rank-1 int64). transformers.js only | |
| // fills `num_logits_to_keep`, so inject it on every forward pass for those. | |
| // The decoder session is "model" for plain LMs, "decoder_model_merged" for VLM decoders. | |
| const decoder = model.sessions?.model ?? model.sessions?.decoder_model_merged; | |
| const inputNames = decoder?.inputNames ?? []; | |
| const empties = Object.entries(emptyInputs ?? {}).filter(([name]) => inputNames.includes(name)); | |
| const needsLogitsToKeep = inputNames.includes("logits_to_keep"); | |
| if (needsLogitsToKeep || empties.length) { | |
| const forward = model.forward.bind(model); | |
| model.forward = (inputs) => { | |
| const extra = {}; | |
| if (needsLogitsToKeep) extra.logits_to_keep = inputs.logits_to_keep ?? int64([1], [1]); | |
| for (const [name, [type, dims]] of empties) if (!inputs[name]) extra[name] = emptyTensor(type, dims); | |
| return forward({ ...inputs, ...extra }); | |
| }; | |
| } | |
| onStatus?.("Warming up"); | |
| await generate([{ role: "user", content: "Hi" }], [], { thinking: false, maxNewTokens: 1 }); | |
| await reset(); | |
| } | |
| async function reset() { | |
| if (cached) { const c = cached; cached = null; await c.cache.dispose(); } | |
| } | |
| function stop() { stopRequested = true; stopper.interrupt(); } | |
| function promptTokens(messages, tools, thinking) { | |
| const out = tokenizer.apply_chat_template(messages, { | |
| tools: tools?.length ? tools : undefined, | |
| add_generation_prompt: true, | |
| tokenize: true, | |
| return_tensor: false, | |
| return_dict: false, | |
| enable_thinking: thinking, | |
| ...(chatTemplate ? { chat_template: chatTemplate } : {}), | |
| }); | |
| return Array.isArray(out[0]) ? out[0] : out; | |
| } | |
| /** | |
| * emit({type:'token', text}) / emit({type:'prefill', done, total}) / | |
| * emit({type:'replace', text}) when the thinking guard rewrites the output so far. | |
| * options.thinkingBudget: max thinking tokens (0 disables the guard). | |
| */ | |
| async function generate(messages, tools, options = {}, emit = null) { | |
| try { | |
| const guard = options.thinking && options.thinkingBudget !== 0 && thinkEndId >= 0; | |
| let r = await run(messages, tools, options, emit, "", guard); | |
| if (r.thinkingCut && !r.stopped) { | |
| // Close the thinking ourselves and let the model continue from there. | |
| const prefix = r.text + THINK_CLOSE[r.thinkingCut]; | |
| emit?.({ type: "replace", text: prefix }); | |
| const r2 = await run(messages, tools, options, emit, prefix, false); | |
| r = { | |
| ...r2, | |
| text: prefix + r2.text, | |
| genTokens: r.genTokens + r2.genTokens, | |
| genMs: r.genMs + r2.genMs, | |
| prefillMs: r.prefillMs + r2.prefillMs, | |
| thinkingCut: r.thinkingCut, | |
| }; | |
| } | |
| return r; | |
| } catch (err) { | |
| await reset().catch(() => {}); // the cache may be half-updated | |
| throw err; | |
| } | |
| } | |
| /** One plain generate() over the whole prompt, no cache kept (see fullPrefill). */ | |
| async function runFull(ids, options, emit, useGuard) { | |
| await reset(); | |
| const n = ids.length; | |
| emit?.({ type: "prefill", done: 0, total: n }); | |
| const t0 = performance.now(); | |
| let tFirst = 0; | |
| const streamer = new TextStreamer(tokenizer, { | |
| skip_prompt: true, | |
| skip_special_tokens: true, | |
| callback_function: (text) => { tFirst ||= performance.now(); emit?.({ type: "token", text }); }, | |
| }); | |
| const maxNew = options.maxNewTokens ?? 2048; | |
| const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null; | |
| const res = await model.generate({ | |
| input_ids: int64(ids, [1, n]), | |
| attention_mask: ones(n), | |
| max_new_tokens: maxNew, | |
| do_sample: (options.temperature ?? 0) > 0, | |
| temperature: options.temperature ?? 0.1, | |
| top_k: options.topK ?? 50, | |
| top_p: options.topP ?? 1.0, | |
| repetition_penalty: options.repetitionPenalty ?? 1.1, | |
| stopping_criteria: guard ? [stopper, guard] : stopper, | |
| streamer, | |
| return_dict_in_generate: true, | |
| }); | |
| const end = performance.now(); | |
| const seq = res.sequences.tolist()[0].map(Number); | |
| res.past_key_values?.dispose?.(); | |
| let gen = seq.slice(n); | |
| const thinkingCut = guard?.reason ?? null; | |
| if (thinkingCut) gen = gen.slice(0, guard.cutAt); | |
| return { | |
| promptTokens: n, reused: 0, newPrefill: n, prefillMs: (tFirst || end) - t0, | |
| thinkingCut, | |
| text: tokenizer.decode(gen, { skip_special_tokens: true }), | |
| stopped: stopRequested, | |
| genTokens: gen.length, | |
| genMs: tFirst ? end - tFirst : 0, | |
| hitLimit: gen.length >= maxNew, | |
| }; | |
| } | |
| async function run(messages, tools, options, emit, assistantPrefix, useGuard) { | |
| stopRequested = false; | |
| stopper.reset(); | |
| const ids = promptTokens(messages, tools, options.thinking); | |
| if (assistantPrefix) ids.push(...tokenizer.encode(assistantPrefix, { add_special_tokens: false })); | |
| const n = ids.length; | |
| if (fullPrefill) return runFull(ids, options, emit, useGuard); | |
| let start = 0; | |
| if (cached && cached.tokens.length < n && cached.tokens.every((t, i) => t === ids[i])) { | |
| start = cached.tokens.length; | |
| } else { | |
| await reset(); | |
| cached = { tokens: [], cache: new DynamicCache() }; | |
| } | |
| const cache = cached.cache; | |
| // Chunked prefill of everything except the last prompt token. | |
| const tPrefill = performance.now(); | |
| let pos = start; | |
| while (pos < n - 1) { | |
| if (stopRequested) break; | |
| const end = Math.min(pos + PREFILL_CHUNK, n - 1); | |
| const out = await model.forward({ | |
| input_ids: int64(ids.slice(pos, end), [1, end - pos]), | |
| attention_mask: ones(end), | |
| past_key_values: cache, | |
| }); | |
| const entries = {}; | |
| for (const k in out) { | |
| if (k.startsWith("present")) entries[pastName(k)] = out[k]; | |
| } | |
| cache.update(entries); | |
| out.logits?.dispose?.(); | |
| pos = end; | |
| cached.tokens = ids.slice(0, pos); | |
| emit?.({ type: "prefill", done: pos - start, total: n - 1 - start }); | |
| } | |
| const prefillMs = performance.now() - tPrefill; | |
| const base = { promptTokens: n, reused: start, newPrefill: pos - start, prefillMs }; | |
| if (stopRequested) return { ...base, text: "", stopped: true, genTokens: 0, genMs: 0, hitLimit: false }; | |
| const streamer = emit ? new TextStreamer(tokenizer, { | |
| skip_prompt: true, | |
| skip_special_tokens: true, | |
| callback_function: (text) => emit({ type: "token", text }), | |
| }) : null; | |
| const maxNew = options.maxNewTokens ?? 2048; | |
| const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null; | |
| const tGen = performance.now(); | |
| const res = await model.generate({ | |
| input_ids: int64(ids, [1, n]), | |
| attention_mask: ones(n), | |
| past_key_values: cache, | |
| max_new_tokens: maxNew, | |
| do_sample: (options.temperature ?? 0) > 0, | |
| temperature: options.temperature ?? 0.1, | |
| top_k: options.topK ?? 50, | |
| top_p: options.topP ?? 1.0, | |
| repetition_penalty: options.repetitionPenalty ?? 1.1, | |
| stopping_criteria: guard ? [stopper, guard] : stopper, | |
| streamer, | |
| return_dict_in_generate: true, | |
| }); | |
| const genMs = performance.now() - tGen; | |
| const seq = res.sequences.tolist()[0].map(Number); | |
| cached.tokens = seq.slice(0, seq.length - 1); // the cache holds all but the last sampled token | |
| let gen = seq.slice(n); | |
| const thinkingCut = guard?.reason ?? null; | |
| if (thinkingCut) gen = gen.slice(0, guard.cutAt); | |
| return { | |
| ...base, | |
| thinkingCut, | |
| text: tokenizer.decode(gen, { skip_special_tokens: true }), | |
| stopped: stopRequested, | |
| genTokens: gen.length, | |
| genMs, | |
| hitLimit: gen.length >= maxNew, | |
| }; | |
| } | |
| return { load, generate, stop, reset }; | |
| } | |