Spaces:
Running
Running
File size: 12,538 Bytes
a8f5a2f 40a5fbc a8f5a2f ea31752 a8f5a2f f4e16d6 ea31752 6b1f716 ea31752 6b1f716 ea31752 6b1f716 a8f5a2f f4e16d6 a8f5a2f 40a5fbc 6b1f716 ea31752 40a5fbc ea31752 40a5fbc a8f5a2f ea31752 a8f5a2f 6b1f716 a8f5a2f 6b1f716 a8f5a2f 40a5fbc a8f5a2f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 | // LFM3 generation engine, independent of where it runs. The browser worker passes
// in the transformers.js web build; tools/agent-e2e.mjs passes the Node build so the
// exact same logic can be tested off-GPU.
//
// Generation = chunked prefill (LiquidAI: "on WebGPU ... prefill in chunks of at most
// 256 tokens") followed by transformers.js generate(). The KV/conv cache is kept
// between calls: if the next prompt starts with exactly the tokens already in the
// cache (the usual case inside one agent run), only the new tokens are prefilled.
const PREFILL_CHUNK = 256;
const LOOP_WINDOW = 40; // a verbatim repeat of this many tokens while thinking = a loop
const LOOP_MIN_TOKENS = 160;
const THINK_CLOSE = {
loop: "\n\nI am going in circles. I'll stop deliberating and take the next concrete step.\n</think>\n",
budget: "\n\nI have thought about this enough. Time to act.\n</think>\n",
};
// Maps a graph output (present_*) to the matching cache input, for every cache kind the
// supported architectures use: attention KV, conv (LFM), SSM (Granite), recurrent (Qwen3.5).
// Mirrors transformers.js' own mapping.
const pastName = (k) => k
.replace("present_ssm", "past_ssm").replace("present_conv", "past_conv")
.replace("present_recurrent", "past_recurrent").replace("present", "past_key_values");
export function createEngine(tf) {
const { AutoTokenizer, AutoModelForCausalLM, DynamicCache, Tensor, TextStreamer, InterruptableStoppingCriteria, StoppingCriteria } = tf;
let tokenizer = null;
let model = null;
let cached = null; // { tokens: number[], cache: DynamicCache }
const stopper = new InterruptableStoppingCriteria();
let stopRequested = false;
let thinkEndId = -1;
// Stops generation while the model is still inside <think> if it starts repeating
// itself verbatim or exceeds the thinking budget. Inactive once </think> appears.
class ThinkingGuard extends StoppingCriteria {
constructor(promptLen, budget) {
super();
this.promptLen = promptLen;
this.budget = budget;
this.scanned = promptLen;
this.closed = false;
this.reason = null;
this.cutAt = -1;
}
_call(input_ids) {
if (this.closed) return [false];
if (this.reason) return [true];
const ids = input_ids[0];
for (let i = this.scanned; i < ids.length; i++) {
if (Number(ids[i]) === thinkEndId) { this.closed = true; return [false]; }
}
this.scanned = ids.length;
const len = ids.length - this.promptLen;
if (len >= this.budget) { this.reason = "budget"; this.cutAt = len; return [true]; }
if (len >= LOOP_MIN_TOKENS && len % 8 === 0) {
const g = ids.slice(this.promptLen).map(Number);
const tailStart = g.length - LOOP_WINDOW;
outer: for (let s = 0; s + LOOP_WINDOW <= tailStart; s++) {
for (let k = 0; k < LOOP_WINDOW; k++) if (g[s + k] !== g[tailStart + k]) continue outer;
this.reason = "loop";
this.cutAt = tailStart; // keep one copy of the repeated passage
return [true];
}
}
return [false];
}
}
const int64 = (arr, dims) => new Tensor("int64", BigInt64Array.from(arr, BigInt), dims);
const ones = (n) => new Tensor("int64", new BigInt64Array(n).fill(1n), [1, n]);
const emptyTensor = (type, dims) => new Tensor(type, type === "int64" ? new BigInt64Array(0) : new Float32Array(0), dims);
// modelOptions: extra from_pretrained options (the browser worker uses it to hand
// onnxruntime the weights as a lazily read, disk-backed buffer).
// chatTemplate: overrides the tokenizer's template. emptyInputs: graph inputs to feed as
// empty tensors on every forward pass ({name: [type, dims]}), e.g. the image/audio inputs
// of a multimodal decoder used for text only.
// fullPrefill: let transformers.js prefill the whole prompt itself and don't reuse the
// cache between calls. Used for vision-language decoders run text-only (Qwen3.5 2B/4B),
// whose multimodal rotary positions only transformers.js' own generate() computes.
let chatTemplate;
let fullPrefill = false;
async function load(modelId, { dtype, device, progress_callback, onStatus, modelOptions, chatTemplate: template, emptyInputs, fullPrefill: full = false } = {}) {
chatTemplate = template;
fullPrefill = full;
tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback });
onStatus?.("Downloading / loading model weights");
model = await AutoModelForCausalLM.from_pretrained(modelId, { dtype, device, progress_callback, ...modelOptions });
const te = tokenizer.encode("</think>", { add_special_tokens: false });
thinkEndId = te.length === 1 ? te[0] : -1;
// Some graphs (LFM3) take `logits_to_keep` (rank-1 int64). transformers.js only
// fills `num_logits_to_keep`, so inject it on every forward pass for those.
// The decoder session is "model" for plain LMs, "decoder_model_merged" for VLM decoders.
const decoder = model.sessions?.model ?? model.sessions?.decoder_model_merged;
const inputNames = decoder?.inputNames ?? [];
const empties = Object.entries(emptyInputs ?? {}).filter(([name]) => inputNames.includes(name));
const needsLogitsToKeep = inputNames.includes("logits_to_keep");
if (needsLogitsToKeep || empties.length) {
const forward = model.forward.bind(model);
model.forward = (inputs) => {
const extra = {};
if (needsLogitsToKeep) extra.logits_to_keep = inputs.logits_to_keep ?? int64([1], [1]);
for (const [name, [type, dims]] of empties) if (!inputs[name]) extra[name] = emptyTensor(type, dims);
return forward({ ...inputs, ...extra });
};
}
onStatus?.("Warming up");
await generate([{ role: "user", content: "Hi" }], [], { thinking: false, maxNewTokens: 1 });
await reset();
}
async function reset() {
if (cached) { const c = cached; cached = null; await c.cache.dispose(); }
}
function stop() { stopRequested = true; stopper.interrupt(); }
function promptTokens(messages, tools, thinking) {
const out = tokenizer.apply_chat_template(messages, {
tools: tools?.length ? tools : undefined,
add_generation_prompt: true,
tokenize: true,
return_tensor: false,
return_dict: false,
enable_thinking: thinking,
...(chatTemplate ? { chat_template: chatTemplate } : {}),
});
return Array.isArray(out[0]) ? out[0] : out;
}
/**
* emit({type:'token', text}) / emit({type:'prefill', done, total}) /
* emit({type:'replace', text}) when the thinking guard rewrites the output so far.
* options.thinkingBudget: max thinking tokens (0 disables the guard).
*/
async function generate(messages, tools, options = {}, emit = null) {
try {
const guard = options.thinking && options.thinkingBudget !== 0 && thinkEndId >= 0;
let r = await run(messages, tools, options, emit, "", guard);
if (r.thinkingCut && !r.stopped) {
// Close the thinking ourselves and let the model continue from there.
const prefix = r.text + THINK_CLOSE[r.thinkingCut];
emit?.({ type: "replace", text: prefix });
const r2 = await run(messages, tools, options, emit, prefix, false);
r = {
...r2,
text: prefix + r2.text,
genTokens: r.genTokens + r2.genTokens,
genMs: r.genMs + r2.genMs,
prefillMs: r.prefillMs + r2.prefillMs,
thinkingCut: r.thinkingCut,
};
}
return r;
} catch (err) {
await reset().catch(() => {}); // the cache may be half-updated
throw err;
}
}
/** One plain generate() over the whole prompt, no cache kept (see fullPrefill). */
async function runFull(ids, options, emit, useGuard) {
await reset();
const n = ids.length;
emit?.({ type: "prefill", done: 0, total: n });
const t0 = performance.now();
let tFirst = 0;
const streamer = new TextStreamer(tokenizer, {
skip_prompt: true,
skip_special_tokens: true,
callback_function: (text) => { tFirst ||= performance.now(); emit?.({ type: "token", text }); },
});
const maxNew = options.maxNewTokens ?? 2048;
const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null;
const res = await model.generate({
input_ids: int64(ids, [1, n]),
attention_mask: ones(n),
max_new_tokens: maxNew,
do_sample: (options.temperature ?? 0) > 0,
temperature: options.temperature ?? 0.1,
top_k: options.topK ?? 50,
top_p: options.topP ?? 1.0,
repetition_penalty: options.repetitionPenalty ?? 1.1,
stopping_criteria: guard ? [stopper, guard] : stopper,
streamer,
return_dict_in_generate: true,
});
const end = performance.now();
const seq = res.sequences.tolist()[0].map(Number);
res.past_key_values?.dispose?.();
let gen = seq.slice(n);
const thinkingCut = guard?.reason ?? null;
if (thinkingCut) gen = gen.slice(0, guard.cutAt);
return {
promptTokens: n, reused: 0, newPrefill: n, prefillMs: (tFirst || end) - t0,
thinkingCut,
text: tokenizer.decode(gen, { skip_special_tokens: true }),
stopped: stopRequested,
genTokens: gen.length,
genMs: tFirst ? end - tFirst : 0,
hitLimit: gen.length >= maxNew,
};
}
async function run(messages, tools, options, emit, assistantPrefix, useGuard) {
stopRequested = false;
stopper.reset();
const ids = promptTokens(messages, tools, options.thinking);
if (assistantPrefix) ids.push(...tokenizer.encode(assistantPrefix, { add_special_tokens: false }));
const n = ids.length;
if (fullPrefill) return runFull(ids, options, emit, useGuard);
let start = 0;
if (cached && cached.tokens.length < n && cached.tokens.every((t, i) => t === ids[i])) {
start = cached.tokens.length;
} else {
await reset();
cached = { tokens: [], cache: new DynamicCache() };
}
const cache = cached.cache;
// Chunked prefill of everything except the last prompt token.
const tPrefill = performance.now();
let pos = start;
while (pos < n - 1) {
if (stopRequested) break;
const end = Math.min(pos + PREFILL_CHUNK, n - 1);
const out = await model.forward({
input_ids: int64(ids.slice(pos, end), [1, end - pos]),
attention_mask: ones(end),
past_key_values: cache,
});
const entries = {};
for (const k in out) {
if (k.startsWith("present")) entries[pastName(k)] = out[k];
}
cache.update(entries);
out.logits?.dispose?.();
pos = end;
cached.tokens = ids.slice(0, pos);
emit?.({ type: "prefill", done: pos - start, total: n - 1 - start });
}
const prefillMs = performance.now() - tPrefill;
const base = { promptTokens: n, reused: start, newPrefill: pos - start, prefillMs };
if (stopRequested) return { ...base, text: "", stopped: true, genTokens: 0, genMs: 0, hitLimit: false };
const streamer = emit ? new TextStreamer(tokenizer, {
skip_prompt: true,
skip_special_tokens: true,
callback_function: (text) => emit({ type: "token", text }),
}) : null;
const maxNew = options.maxNewTokens ?? 2048;
const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null;
const tGen = performance.now();
const res = await model.generate({
input_ids: int64(ids, [1, n]),
attention_mask: ones(n),
past_key_values: cache,
max_new_tokens: maxNew,
do_sample: (options.temperature ?? 0) > 0,
temperature: options.temperature ?? 0.1,
top_k: options.topK ?? 50,
top_p: options.topP ?? 1.0,
repetition_penalty: options.repetitionPenalty ?? 1.1,
stopping_criteria: guard ? [stopper, guard] : stopper,
streamer,
return_dict_in_generate: true,
});
const genMs = performance.now() - tGen;
const seq = res.sequences.tolist()[0].map(Number);
cached.tokens = seq.slice(0, seq.length - 1); // the cache holds all but the last sampled token
let gen = seq.slice(n);
const thinkingCut = guard?.reason ?? null;
if (thinkingCut) gen = gen.slice(0, guard.cutAt);
return {
...base,
thinkingCut,
text: tokenizer.decode(gen, { skip_special_tokens: true }),
stopped: stopRequested,
genTokens: gen.length,
genMs,
hitLimit: gen.length >= maxNew,
};
}
return { load, generate, stop, reset };
}
|