// LFM3 generation engine, independent of where it runs. The browser worker passes // in the transformers.js web build; tools/agent-e2e.mjs passes the Node build so the // exact same logic can be tested off-GPU. // // Generation = chunked prefill (LiquidAI: "on WebGPU ... prefill in chunks of at most // 256 tokens") followed by transformers.js generate(). The KV/conv cache is kept // between calls: if the next prompt starts with exactly the tokens already in the // cache (the usual case inside one agent run), only the new tokens are prefilled. const PREFILL_CHUNK = 256; const LOOP_WINDOW = 40; // a verbatim repeat of this many tokens while thinking = a loop const LOOP_MIN_TOKENS = 160; const THINK_CLOSE = { loop: "\n\nI am going in circles. I'll stop deliberating and take the next concrete step.\n\n", budget: "\n\nI have thought about this enough. Time to act.\n\n", }; // Maps a graph output (present_*) to the matching cache input, for every cache kind the // supported architectures use: attention KV, conv (LFM), SSM (Granite), recurrent (Qwen3.5). // Mirrors transformers.js' own mapping. const pastName = (k) => k .replace("present_ssm", "past_ssm").replace("present_conv", "past_conv") .replace("present_recurrent", "past_recurrent").replace("present", "past_key_values"); export function createEngine(tf) { const { AutoTokenizer, AutoModelForCausalLM, DynamicCache, Tensor, TextStreamer, InterruptableStoppingCriteria, StoppingCriteria } = tf; let tokenizer = null; let model = null; let cached = null; // { tokens: number[], cache: DynamicCache } const stopper = new InterruptableStoppingCriteria(); let stopRequested = false; let thinkEndId = -1; // Stops generation while the model is still inside if it starts repeating // itself verbatim or exceeds the thinking budget. Inactive once appears. class ThinkingGuard extends StoppingCriteria { constructor(promptLen, budget) { super(); this.promptLen = promptLen; this.budget = budget; this.scanned = promptLen; this.closed = false; this.reason = null; this.cutAt = -1; } _call(input_ids) { if (this.closed) return [false]; if (this.reason) return [true]; const ids = input_ids[0]; for (let i = this.scanned; i < ids.length; i++) { if (Number(ids[i]) === thinkEndId) { this.closed = true; return [false]; } } this.scanned = ids.length; const len = ids.length - this.promptLen; if (len >= this.budget) { this.reason = "budget"; this.cutAt = len; return [true]; } if (len >= LOOP_MIN_TOKENS && len % 8 === 0) { const g = ids.slice(this.promptLen).map(Number); const tailStart = g.length - LOOP_WINDOW; outer: for (let s = 0; s + LOOP_WINDOW <= tailStart; s++) { for (let k = 0; k < LOOP_WINDOW; k++) if (g[s + k] !== g[tailStart + k]) continue outer; this.reason = "loop"; this.cutAt = tailStart; // keep one copy of the repeated passage return [true]; } } return [false]; } } const int64 = (arr, dims) => new Tensor("int64", BigInt64Array.from(arr, BigInt), dims); const ones = (n) => new Tensor("int64", new BigInt64Array(n).fill(1n), [1, n]); const emptyTensor = (type, dims) => new Tensor(type, type === "int64" ? new BigInt64Array(0) : new Float32Array(0), dims); // modelOptions: extra from_pretrained options (the browser worker uses it to hand // onnxruntime the weights as a lazily read, disk-backed buffer). // chatTemplate: overrides the tokenizer's template. emptyInputs: graph inputs to feed as // empty tensors on every forward pass ({name: [type, dims]}), e.g. the image/audio inputs // of a multimodal decoder used for text only. // fullPrefill: let transformers.js prefill the whole prompt itself and don't reuse the // cache between calls. Used for vision-language decoders run text-only (Qwen3.5 2B/4B), // whose multimodal rotary positions only transformers.js' own generate() computes. let chatTemplate; let fullPrefill = false; async function load(modelId, { dtype, device, progress_callback, onStatus, modelOptions, chatTemplate: template, emptyInputs, fullPrefill: full = false } = {}) { chatTemplate = template; fullPrefill = full; tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback }); onStatus?.("Downloading / loading model weights"); model = await AutoModelForCausalLM.from_pretrained(modelId, { dtype, device, progress_callback, ...modelOptions }); const te = tokenizer.encode("", { add_special_tokens: false }); thinkEndId = te.length === 1 ? te[0] : -1; // Some graphs (LFM3) take `logits_to_keep` (rank-1 int64). transformers.js only // fills `num_logits_to_keep`, so inject it on every forward pass for those. // The decoder session is "model" for plain LMs, "decoder_model_merged" for VLM decoders. const decoder = model.sessions?.model ?? model.sessions?.decoder_model_merged; const inputNames = decoder?.inputNames ?? []; const empties = Object.entries(emptyInputs ?? {}).filter(([name]) => inputNames.includes(name)); const needsLogitsToKeep = inputNames.includes("logits_to_keep"); if (needsLogitsToKeep || empties.length) { const forward = model.forward.bind(model); model.forward = (inputs) => { const extra = {}; if (needsLogitsToKeep) extra.logits_to_keep = inputs.logits_to_keep ?? int64([1], [1]); for (const [name, [type, dims]] of empties) if (!inputs[name]) extra[name] = emptyTensor(type, dims); return forward({ ...inputs, ...extra }); }; } onStatus?.("Warming up"); await generate([{ role: "user", content: "Hi" }], [], { thinking: false, maxNewTokens: 1 }); await reset(); } async function reset() { if (cached) { const c = cached; cached = null; await c.cache.dispose(); } } function stop() { stopRequested = true; stopper.interrupt(); } function promptTokens(messages, tools, thinking) { const out = tokenizer.apply_chat_template(messages, { tools: tools?.length ? tools : undefined, add_generation_prompt: true, tokenize: true, return_tensor: false, return_dict: false, enable_thinking: thinking, ...(chatTemplate ? { chat_template: chatTemplate } : {}), }); return Array.isArray(out[0]) ? out[0] : out; } /** * emit({type:'token', text}) / emit({type:'prefill', done, total}) / * emit({type:'replace', text}) when the thinking guard rewrites the output so far. * options.thinkingBudget: max thinking tokens (0 disables the guard). */ async function generate(messages, tools, options = {}, emit = null) { try { const guard = options.thinking && options.thinkingBudget !== 0 && thinkEndId >= 0; let r = await run(messages, tools, options, emit, "", guard); if (r.thinkingCut && !r.stopped) { // Close the thinking ourselves and let the model continue from there. const prefix = r.text + THINK_CLOSE[r.thinkingCut]; emit?.({ type: "replace", text: prefix }); const r2 = await run(messages, tools, options, emit, prefix, false); r = { ...r2, text: prefix + r2.text, genTokens: r.genTokens + r2.genTokens, genMs: r.genMs + r2.genMs, prefillMs: r.prefillMs + r2.prefillMs, thinkingCut: r.thinkingCut, }; } return r; } catch (err) { await reset().catch(() => {}); // the cache may be half-updated throw err; } } /** One plain generate() over the whole prompt, no cache kept (see fullPrefill). */ async function runFull(ids, options, emit, useGuard) { await reset(); const n = ids.length; emit?.({ type: "prefill", done: 0, total: n }); const t0 = performance.now(); let tFirst = 0; const streamer = new TextStreamer(tokenizer, { skip_prompt: true, skip_special_tokens: true, callback_function: (text) => { tFirst ||= performance.now(); emit?.({ type: "token", text }); }, }); const maxNew = options.maxNewTokens ?? 2048; const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null; const res = await model.generate({ input_ids: int64(ids, [1, n]), attention_mask: ones(n), max_new_tokens: maxNew, do_sample: (options.temperature ?? 0) > 0, temperature: options.temperature ?? 0.1, top_k: options.topK ?? 50, top_p: options.topP ?? 1.0, repetition_penalty: options.repetitionPenalty ?? 1.1, stopping_criteria: guard ? [stopper, guard] : stopper, streamer, return_dict_in_generate: true, }); const end = performance.now(); const seq = res.sequences.tolist()[0].map(Number); res.past_key_values?.dispose?.(); let gen = seq.slice(n); const thinkingCut = guard?.reason ?? null; if (thinkingCut) gen = gen.slice(0, guard.cutAt); return { promptTokens: n, reused: 0, newPrefill: n, prefillMs: (tFirst || end) - t0, thinkingCut, text: tokenizer.decode(gen, { skip_special_tokens: true }), stopped: stopRequested, genTokens: gen.length, genMs: tFirst ? end - tFirst : 0, hitLimit: gen.length >= maxNew, }; } async function run(messages, tools, options, emit, assistantPrefix, useGuard) { stopRequested = false; stopper.reset(); const ids = promptTokens(messages, tools, options.thinking); if (assistantPrefix) ids.push(...tokenizer.encode(assistantPrefix, { add_special_tokens: false })); const n = ids.length; if (fullPrefill) return runFull(ids, options, emit, useGuard); let start = 0; if (cached && cached.tokens.length < n && cached.tokens.every((t, i) => t === ids[i])) { start = cached.tokens.length; } else { await reset(); cached = { tokens: [], cache: new DynamicCache() }; } const cache = cached.cache; // Chunked prefill of everything except the last prompt token. const tPrefill = performance.now(); let pos = start; while (pos < n - 1) { if (stopRequested) break; const end = Math.min(pos + PREFILL_CHUNK, n - 1); const out = await model.forward({ input_ids: int64(ids.slice(pos, end), [1, end - pos]), attention_mask: ones(end), past_key_values: cache, }); const entries = {}; for (const k in out) { if (k.startsWith("present")) entries[pastName(k)] = out[k]; } cache.update(entries); out.logits?.dispose?.(); pos = end; cached.tokens = ids.slice(0, pos); emit?.({ type: "prefill", done: pos - start, total: n - 1 - start }); } const prefillMs = performance.now() - tPrefill; const base = { promptTokens: n, reused: start, newPrefill: pos - start, prefillMs }; if (stopRequested) return { ...base, text: "", stopped: true, genTokens: 0, genMs: 0, hitLimit: false }; const streamer = emit ? new TextStreamer(tokenizer, { skip_prompt: true, skip_special_tokens: true, callback_function: (text) => emit({ type: "token", text }), }) : null; const maxNew = options.maxNewTokens ?? 2048; const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null; const tGen = performance.now(); const res = await model.generate({ input_ids: int64(ids, [1, n]), attention_mask: ones(n), past_key_values: cache, max_new_tokens: maxNew, do_sample: (options.temperature ?? 0) > 0, temperature: options.temperature ?? 0.1, top_k: options.topK ?? 50, top_p: options.topP ?? 1.0, repetition_penalty: options.repetitionPenalty ?? 1.1, stopping_criteria: guard ? [stopper, guard] : stopper, streamer, return_dict_in_generate: true, }); const genMs = performance.now() - tGen; const seq = res.sequences.tolist()[0].map(Number); cached.tokens = seq.slice(0, seq.length - 1); // the cache holds all but the last sampled token let gen = seq.slice(n); const thinkingCut = guard?.reason ?? null; if (thinkingCut) gen = gen.slice(0, guard.cutAt); return { ...base, thinkingCut, text: tokenizer.decode(gen, { skip_special_tokens: true }), stopped: stopRequested, genTokens: gen.length, genMs, hitLimit: gen.length >= maxNew, }; } return { load, generate, stop, reset }; }