LocalWebAgent / js /llm-engine.js
ereniko's picture
v0.7.0: Qwen3.5 2B/4B, SmolLM3 3B, Llama 3.2 3B, Jan-nano 4B; drop Qwen3 4B
6b1f716 unverified
Raw History Blame Contribute Delete
12.5 kB
// LFM3 generation engine, independent of where it runs. The browser worker passes
// in the transformers.js web build; tools/agent-e2e.mjs passes the Node build so the
// exact same logic can be tested off-GPU.
//
// Generation = chunked prefill (LiquidAI: "on WebGPU ... prefill in chunks of at most
// 256 tokens") followed by transformers.js generate(). The KV/conv cache is kept
// between calls: if the next prompt starts with exactly the tokens already in the
// cache (the usual case inside one agent run), only the new tokens are prefilled.
const PREFILL_CHUNK = 256;
const LOOP_WINDOW = 40; // a verbatim repeat of this many tokens while thinking = a loop
const LOOP_MIN_TOKENS = 160;
const THINK_CLOSE = {
loop: "\n\nI am going in circles. I'll stop deliberating and take the next concrete step.\n</think>\n",
budget: "\n\nI have thought about this enough. Time to act.\n</think>\n",
};
// Maps a graph output (present_*) to the matching cache input, for every cache kind the
// supported architectures use: attention KV, conv (LFM), SSM (Granite), recurrent (Qwen3.5).
// Mirrors transformers.js' own mapping.
const pastName = (k) => k
.replace("present_ssm", "past_ssm").replace("present_conv", "past_conv")
.replace("present_recurrent", "past_recurrent").replace("present", "past_key_values");
export function createEngine(tf) {
const { AutoTokenizer, AutoModelForCausalLM, DynamicCache, Tensor, TextStreamer, InterruptableStoppingCriteria, StoppingCriteria } = tf;
let tokenizer = null;
let model = null;
let cached = null; // { tokens: number[], cache: DynamicCache }
const stopper = new InterruptableStoppingCriteria();
let stopRequested = false;
let thinkEndId = -1;
// Stops generation while the model is still inside <think> if it starts repeating
// itself verbatim or exceeds the thinking budget. Inactive once </think> appears.
class ThinkingGuard extends StoppingCriteria {
constructor(promptLen, budget) {
super();
this.promptLen = promptLen;
this.budget = budget;
this.scanned = promptLen;
this.closed = false;
this.reason = null;
this.cutAt = -1;
}
_call(input_ids) {
if (this.closed) return [false];
if (this.reason) return [true];
const ids = input_ids[0];
for (let i = this.scanned; i < ids.length; i++) {
if (Number(ids[i]) === thinkEndId) { this.closed = true; return [false]; }
}
this.scanned = ids.length;
const len = ids.length - this.promptLen;
if (len >= this.budget) { this.reason = "budget"; this.cutAt = len; return [true]; }
if (len >= LOOP_MIN_TOKENS && len % 8 === 0) {
const g = ids.slice(this.promptLen).map(Number);
const tailStart = g.length - LOOP_WINDOW;
outer: for (let s = 0; s + LOOP_WINDOW <= tailStart; s++) {
for (let k = 0; k < LOOP_WINDOW; k++) if (g[s + k] !== g[tailStart + k]) continue outer;
this.reason = "loop";
this.cutAt = tailStart; // keep one copy of the repeated passage
return [true];
}
}
return [false];
}
}
const int64 = (arr, dims) => new Tensor("int64", BigInt64Array.from(arr, BigInt), dims);
const ones = (n) => new Tensor("int64", new BigInt64Array(n).fill(1n), [1, n]);
const emptyTensor = (type, dims) => new Tensor(type, type === "int64" ? new BigInt64Array(0) : new Float32Array(0), dims);
// modelOptions: extra from_pretrained options (the browser worker uses it to hand
// onnxruntime the weights as a lazily read, disk-backed buffer).
// chatTemplate: overrides the tokenizer's template. emptyInputs: graph inputs to feed as
// empty tensors on every forward pass ({name: [type, dims]}), e.g. the image/audio inputs
// of a multimodal decoder used for text only.
// fullPrefill: let transformers.js prefill the whole prompt itself and don't reuse the
// cache between calls. Used for vision-language decoders run text-only (Qwen3.5 2B/4B),
// whose multimodal rotary positions only transformers.js' own generate() computes.
let chatTemplate;
let fullPrefill = false;
async function load(modelId, { dtype, device, progress_callback, onStatus, modelOptions, chatTemplate: template, emptyInputs, fullPrefill: full = false } = {}) {
chatTemplate = template;
fullPrefill = full;
tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback });
onStatus?.("Downloading / loading model weights");
model = await AutoModelForCausalLM.from_pretrained(modelId, { dtype, device, progress_callback, ...modelOptions });
const te = tokenizer.encode("</think>", { add_special_tokens: false });
thinkEndId = te.length === 1 ? te[0] : -1;
// Some graphs (LFM3) take `logits_to_keep` (rank-1 int64). transformers.js only
// fills `num_logits_to_keep`, so inject it on every forward pass for those.
// The decoder session is "model" for plain LMs, "decoder_model_merged" for VLM decoders.
const decoder = model.sessions?.model ?? model.sessions?.decoder_model_merged;
const inputNames = decoder?.inputNames ?? [];
const empties = Object.entries(emptyInputs ?? {}).filter(([name]) => inputNames.includes(name));
const needsLogitsToKeep = inputNames.includes("logits_to_keep");
if (needsLogitsToKeep || empties.length) {
const forward = model.forward.bind(model);
model.forward = (inputs) => {
const extra = {};
if (needsLogitsToKeep) extra.logits_to_keep = inputs.logits_to_keep ?? int64([1], [1]);
for (const [name, [type, dims]] of empties) if (!inputs[name]) extra[name] = emptyTensor(type, dims);
return forward({ ...inputs, ...extra });
};
}
onStatus?.("Warming up");
await generate([{ role: "user", content: "Hi" }], [], { thinking: false, maxNewTokens: 1 });
await reset();
}
async function reset() {
if (cached) { const c = cached; cached = null; await c.cache.dispose(); }
}
function stop() { stopRequested = true; stopper.interrupt(); }
function promptTokens(messages, tools, thinking) {
const out = tokenizer.apply_chat_template(messages, {
tools: tools?.length ? tools : undefined,
add_generation_prompt: true,
tokenize: true,
return_tensor: false,
return_dict: false,
enable_thinking: thinking,
...(chatTemplate ? { chat_template: chatTemplate } : {}),
});
return Array.isArray(out[0]) ? out[0] : out;
}
/**
* emit({type:'token', text}) / emit({type:'prefill', done, total}) /
* emit({type:'replace', text}) when the thinking guard rewrites the output so far.
* options.thinkingBudget: max thinking tokens (0 disables the guard).
*/
async function generate(messages, tools, options = {}, emit = null) {
try {
const guard = options.thinking && options.thinkingBudget !== 0 && thinkEndId >= 0;
let r = await run(messages, tools, options, emit, "", guard);
if (r.thinkingCut && !r.stopped) {
// Close the thinking ourselves and let the model continue from there.
const prefix = r.text + THINK_CLOSE[r.thinkingCut];
emit?.({ type: "replace", text: prefix });
const r2 = await run(messages, tools, options, emit, prefix, false);
r = {
...r2,
text: prefix + r2.text,
genTokens: r.genTokens + r2.genTokens,
genMs: r.genMs + r2.genMs,
prefillMs: r.prefillMs + r2.prefillMs,
thinkingCut: r.thinkingCut,
};
}
return r;
} catch (err) {
await reset().catch(() => {}); // the cache may be half-updated
throw err;
}
}
/** One plain generate() over the whole prompt, no cache kept (see fullPrefill). */
async function runFull(ids, options, emit, useGuard) {
await reset();
const n = ids.length;
emit?.({ type: "prefill", done: 0, total: n });
const t0 = performance.now();
let tFirst = 0;
const streamer = new TextStreamer(tokenizer, {
skip_prompt: true,
skip_special_tokens: true,
callback_function: (text) => { tFirst ||= performance.now(); emit?.({ type: "token", text }); },
});
const maxNew = options.maxNewTokens ?? 2048;
const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null;
const res = await model.generate({
input_ids: int64(ids, [1, n]),
attention_mask: ones(n),
max_new_tokens: maxNew,
do_sample: (options.temperature ?? 0) > 0,
temperature: options.temperature ?? 0.1,
top_k: options.topK ?? 50,
top_p: options.topP ?? 1.0,
repetition_penalty: options.repetitionPenalty ?? 1.1,
stopping_criteria: guard ? [stopper, guard] : stopper,
streamer,
return_dict_in_generate: true,
});
const end = performance.now();
const seq = res.sequences.tolist()[0].map(Number);
res.past_key_values?.dispose?.();
let gen = seq.slice(n);
const thinkingCut = guard?.reason ?? null;
if (thinkingCut) gen = gen.slice(0, guard.cutAt);
return {
promptTokens: n, reused: 0, newPrefill: n, prefillMs: (tFirst || end) - t0,
thinkingCut,
text: tokenizer.decode(gen, { skip_special_tokens: true }),
stopped: stopRequested,
genTokens: gen.length,
genMs: tFirst ? end - tFirst : 0,
hitLimit: gen.length >= maxNew,
};
}
async function run(messages, tools, options, emit, assistantPrefix, useGuard) {
stopRequested = false;
stopper.reset();
const ids = promptTokens(messages, tools, options.thinking);
if (assistantPrefix) ids.push(...tokenizer.encode(assistantPrefix, { add_special_tokens: false }));
const n = ids.length;
if (fullPrefill) return runFull(ids, options, emit, useGuard);
let start = 0;
if (cached && cached.tokens.length < n && cached.tokens.every((t, i) => t === ids[i])) {
start = cached.tokens.length;
} else {
await reset();
cached = { tokens: [], cache: new DynamicCache() };
}
const cache = cached.cache;
// Chunked prefill of everything except the last prompt token.
const tPrefill = performance.now();
let pos = start;
while (pos < n - 1) {
if (stopRequested) break;
const end = Math.min(pos + PREFILL_CHUNK, n - 1);
const out = await model.forward({
input_ids: int64(ids.slice(pos, end), [1, end - pos]),
attention_mask: ones(end),
past_key_values: cache,
});
const entries = {};
for (const k in out) {
if (k.startsWith("present")) entries[pastName(k)] = out[k];
}
cache.update(entries);
out.logits?.dispose?.();
pos = end;
cached.tokens = ids.slice(0, pos);
emit?.({ type: "prefill", done: pos - start, total: n - 1 - start });
}
const prefillMs = performance.now() - tPrefill;
const base = { promptTokens: n, reused: start, newPrefill: pos - start, prefillMs };
if (stopRequested) return { ...base, text: "", stopped: true, genTokens: 0, genMs: 0, hitLimit: false };
const streamer = emit ? new TextStreamer(tokenizer, {
skip_prompt: true,
skip_special_tokens: true,
callback_function: (text) => emit({ type: "token", text }),
}) : null;
const maxNew = options.maxNewTokens ?? 2048;
const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null;
const tGen = performance.now();
const res = await model.generate({
input_ids: int64(ids, [1, n]),
attention_mask: ones(n),
past_key_values: cache,
max_new_tokens: maxNew,
do_sample: (options.temperature ?? 0) > 0,
temperature: options.temperature ?? 0.1,
top_k: options.topK ?? 50,
top_p: options.topP ?? 1.0,
repetition_penalty: options.repetitionPenalty ?? 1.1,
stopping_criteria: guard ? [stopper, guard] : stopper,
streamer,
return_dict_in_generate: true,
});
const genMs = performance.now() - tGen;
const seq = res.sequences.tolist()[0].map(Number);
cached.tokens = seq.slice(0, seq.length - 1); // the cache holds all but the last sampled token
let gen = seq.slice(n);
const thinkingCut = guard?.reason ?? null;
if (thinkingCut) gen = gen.slice(0, guard.cutAt);
return {
...base,
thinkingCut,
text: tokenizer.decode(gen, { skip_special_tokens: true }),
stopped: stopRequested,
genTokens: gen.length,
genMs,
hitLimit: gen.length >= maxNew,
};
}
return { load, generate, stop, reset };
}