File size: 12,538 Bytes
a8f5a2f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40a5fbc
 
 
 
 
 
 
a8f5a2f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ea31752
a8f5a2f
f4e16d6
 
ea31752
 
 
6b1f716
 
 
ea31752
6b1f716
 
ea31752
6b1f716
a8f5a2f
 
f4e16d6
a8f5a2f
 
40a5fbc
 
6b1f716
 
 
ea31752
 
 
40a5fbc
ea31752
 
 
 
 
 
40a5fbc
a8f5a2f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ea31752
a8f5a2f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6b1f716
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a8f5a2f
 
 
 
 
 
 
6b1f716
 
a8f5a2f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40a5fbc
a8f5a2f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
// LFM3 generation engine, independent of where it runs. The browser worker passes
// in the transformers.js web build; tools/agent-e2e.mjs passes the Node build so the
// exact same logic can be tested off-GPU.
//
// Generation = chunked prefill (LiquidAI: "on WebGPU ... prefill in chunks of at most
// 256 tokens") followed by transformers.js generate(). The KV/conv cache is kept
// between calls: if the next prompt starts with exactly the tokens already in the
// cache (the usual case inside one agent run), only the new tokens are prefilled.

const PREFILL_CHUNK = 256;
const LOOP_WINDOW = 40;   // a verbatim repeat of this many tokens while thinking = a loop
const LOOP_MIN_TOKENS = 160;
const THINK_CLOSE = {
  loop: "\n\nI am going in circles. I'll stop deliberating and take the next concrete step.\n</think>\n",
  budget: "\n\nI have thought about this enough. Time to act.\n</think>\n",
};

// Maps a graph output (present_*) to the matching cache input, for every cache kind the
// supported architectures use: attention KV, conv (LFM), SSM (Granite), recurrent (Qwen3.5).
// Mirrors transformers.js' own mapping.
const pastName = (k) => k
  .replace("present_ssm", "past_ssm").replace("present_conv", "past_conv")
  .replace("present_recurrent", "past_recurrent").replace("present", "past_key_values");

export function createEngine(tf) {
  const { AutoTokenizer, AutoModelForCausalLM, DynamicCache, Tensor, TextStreamer, InterruptableStoppingCriteria, StoppingCriteria } = tf;
  let tokenizer = null;
  let model = null;
  let cached = null; // { tokens: number[], cache: DynamicCache }
  const stopper = new InterruptableStoppingCriteria();
  let stopRequested = false;

  let thinkEndId = -1;

  // Stops generation while the model is still inside <think> if it starts repeating
  // itself verbatim or exceeds the thinking budget. Inactive once </think> appears.
  class ThinkingGuard extends StoppingCriteria {
    constructor(promptLen, budget) {
      super();
      this.promptLen = promptLen;
      this.budget = budget;
      this.scanned = promptLen;
      this.closed = false;
      this.reason = null;
      this.cutAt = -1;
    }
    _call(input_ids) {
      if (this.closed) return [false];
      if (this.reason) return [true];
      const ids = input_ids[0];
      for (let i = this.scanned; i < ids.length; i++) {
        if (Number(ids[i]) === thinkEndId) { this.closed = true; return [false]; }
      }
      this.scanned = ids.length;
      const len = ids.length - this.promptLen;
      if (len >= this.budget) { this.reason = "budget"; this.cutAt = len; return [true]; }
      if (len >= LOOP_MIN_TOKENS && len % 8 === 0) {
        const g = ids.slice(this.promptLen).map(Number);
        const tailStart = g.length - LOOP_WINDOW;
        outer: for (let s = 0; s + LOOP_WINDOW <= tailStart; s++) {
          for (let k = 0; k < LOOP_WINDOW; k++) if (g[s + k] !== g[tailStart + k]) continue outer;
          this.reason = "loop";
          this.cutAt = tailStart; // keep one copy of the repeated passage
          return [true];
        }
      }
      return [false];
    }
  }

  const int64 = (arr, dims) => new Tensor("int64", BigInt64Array.from(arr, BigInt), dims);
  const ones = (n) => new Tensor("int64", new BigInt64Array(n).fill(1n), [1, n]);
  const emptyTensor = (type, dims) => new Tensor(type, type === "int64" ? new BigInt64Array(0) : new Float32Array(0), dims);

  // modelOptions: extra from_pretrained options (the browser worker uses it to hand
  // onnxruntime the weights as a lazily read, disk-backed buffer).
  // chatTemplate: overrides the tokenizer's template. emptyInputs: graph inputs to feed as
  // empty tensors on every forward pass ({name: [type, dims]}), e.g. the image/audio inputs
  // of a multimodal decoder used for text only.
  // fullPrefill: let transformers.js prefill the whole prompt itself and don't reuse the
  // cache between calls. Used for vision-language decoders run text-only (Qwen3.5 2B/4B),
  // whose multimodal rotary positions only transformers.js' own generate() computes.
  let chatTemplate;
  let fullPrefill = false;
  async function load(modelId, { dtype, device, progress_callback, onStatus, modelOptions, chatTemplate: template, emptyInputs, fullPrefill: full = false } = {}) {
    chatTemplate = template;
    fullPrefill = full;
    tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback });
    onStatus?.("Downloading / loading model weights");
    model = await AutoModelForCausalLM.from_pretrained(modelId, { dtype, device, progress_callback, ...modelOptions });
    const te = tokenizer.encode("</think>", { add_special_tokens: false });
    thinkEndId = te.length === 1 ? te[0] : -1;
    // Some graphs (LFM3) take `logits_to_keep` (rank-1 int64). transformers.js only
    // fills `num_logits_to_keep`, so inject it on every forward pass for those.
    // The decoder session is "model" for plain LMs, "decoder_model_merged" for VLM decoders.
    const decoder = model.sessions?.model ?? model.sessions?.decoder_model_merged;
    const inputNames = decoder?.inputNames ?? [];
    const empties = Object.entries(emptyInputs ?? {}).filter(([name]) => inputNames.includes(name));
    const needsLogitsToKeep = inputNames.includes("logits_to_keep");
    if (needsLogitsToKeep || empties.length) {
      const forward = model.forward.bind(model);
      model.forward = (inputs) => {
        const extra = {};
        if (needsLogitsToKeep) extra.logits_to_keep = inputs.logits_to_keep ?? int64([1], [1]);
        for (const [name, [type, dims]] of empties) if (!inputs[name]) extra[name] = emptyTensor(type, dims);
        return forward({ ...inputs, ...extra });
      };
    }
    onStatus?.("Warming up");
    await generate([{ role: "user", content: "Hi" }], [], { thinking: false, maxNewTokens: 1 });
    await reset();
  }

  async function reset() {
    if (cached) { const c = cached; cached = null; await c.cache.dispose(); }
  }

  function stop() { stopRequested = true; stopper.interrupt(); }

  function promptTokens(messages, tools, thinking) {
    const out = tokenizer.apply_chat_template(messages, {
      tools: tools?.length ? tools : undefined,
      add_generation_prompt: true,
      tokenize: true,
      return_tensor: false,
      return_dict: false,
      enable_thinking: thinking,
      ...(chatTemplate ? { chat_template: chatTemplate } : {}),
    });
    return Array.isArray(out[0]) ? out[0] : out;
  }

  /**
   * emit({type:'token', text}) / emit({type:'prefill', done, total}) /
   * emit({type:'replace', text}) when the thinking guard rewrites the output so far.
   * options.thinkingBudget: max thinking tokens (0 disables the guard).
   */
  async function generate(messages, tools, options = {}, emit = null) {
    try {
      const guard = options.thinking && options.thinkingBudget !== 0 && thinkEndId >= 0;
      let r = await run(messages, tools, options, emit, "", guard);
      if (r.thinkingCut && !r.stopped) {
        // Close the thinking ourselves and let the model continue from there.
        const prefix = r.text + THINK_CLOSE[r.thinkingCut];
        emit?.({ type: "replace", text: prefix });
        const r2 = await run(messages, tools, options, emit, prefix, false);
        r = {
          ...r2,
          text: prefix + r2.text,
          genTokens: r.genTokens + r2.genTokens,
          genMs: r.genMs + r2.genMs,
          prefillMs: r.prefillMs + r2.prefillMs,
          thinkingCut: r.thinkingCut,
        };
      }
      return r;
    } catch (err) {
      await reset().catch(() => {}); // the cache may be half-updated
      throw err;
    }
  }

  /** One plain generate() over the whole prompt, no cache kept (see fullPrefill). */
  async function runFull(ids, options, emit, useGuard) {
    await reset();
    const n = ids.length;
    emit?.({ type: "prefill", done: 0, total: n });
    const t0 = performance.now();
    let tFirst = 0;
    const streamer = new TextStreamer(tokenizer, {
      skip_prompt: true,
      skip_special_tokens: true,
      callback_function: (text) => { tFirst ||= performance.now(); emit?.({ type: "token", text }); },
    });
    const maxNew = options.maxNewTokens ?? 2048;
    const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null;
    const res = await model.generate({
      input_ids: int64(ids, [1, n]),
      attention_mask: ones(n),
      max_new_tokens: maxNew,
      do_sample: (options.temperature ?? 0) > 0,
      temperature: options.temperature ?? 0.1,
      top_k: options.topK ?? 50,
      top_p: options.topP ?? 1.0,
      repetition_penalty: options.repetitionPenalty ?? 1.1,
      stopping_criteria: guard ? [stopper, guard] : stopper,
      streamer,
      return_dict_in_generate: true,
    });
    const end = performance.now();
    const seq = res.sequences.tolist()[0].map(Number);
    res.past_key_values?.dispose?.();
    let gen = seq.slice(n);
    const thinkingCut = guard?.reason ?? null;
    if (thinkingCut) gen = gen.slice(0, guard.cutAt);
    return {
      promptTokens: n, reused: 0, newPrefill: n, prefillMs: (tFirst || end) - t0,
      thinkingCut,
      text: tokenizer.decode(gen, { skip_special_tokens: true }),
      stopped: stopRequested,
      genTokens: gen.length,
      genMs: tFirst ? end - tFirst : 0,
      hitLimit: gen.length >= maxNew,
    };
  }

  async function run(messages, tools, options, emit, assistantPrefix, useGuard) {
    stopRequested = false;
    stopper.reset();
    const ids = promptTokens(messages, tools, options.thinking);
    if (assistantPrefix) ids.push(...tokenizer.encode(assistantPrefix, { add_special_tokens: false }));
    const n = ids.length;

    if (fullPrefill) return runFull(ids, options, emit, useGuard);

    let start = 0;
    if (cached && cached.tokens.length < n && cached.tokens.every((t, i) => t === ids[i])) {
      start = cached.tokens.length;
    } else {
      await reset();
      cached = { tokens: [], cache: new DynamicCache() };
    }
    const cache = cached.cache;

    // Chunked prefill of everything except the last prompt token.
    const tPrefill = performance.now();
    let pos = start;
    while (pos < n - 1) {
      if (stopRequested) break;
      const end = Math.min(pos + PREFILL_CHUNK, n - 1);
      const out = await model.forward({
        input_ids: int64(ids.slice(pos, end), [1, end - pos]),
        attention_mask: ones(end),
        past_key_values: cache,
      });
      const entries = {};
      for (const k in out) {
        if (k.startsWith("present")) entries[pastName(k)] = out[k];
      }
      cache.update(entries);
      out.logits?.dispose?.();
      pos = end;
      cached.tokens = ids.slice(0, pos);
      emit?.({ type: "prefill", done: pos - start, total: n - 1 - start });
    }
    const prefillMs = performance.now() - tPrefill;
    const base = { promptTokens: n, reused: start, newPrefill: pos - start, prefillMs };
    if (stopRequested) return { ...base, text: "", stopped: true, genTokens: 0, genMs: 0, hitLimit: false };

    const streamer = emit ? new TextStreamer(tokenizer, {
      skip_prompt: true,
      skip_special_tokens: true,
      callback_function: (text) => emit({ type: "token", text }),
    }) : null;
    const maxNew = options.maxNewTokens ?? 2048;
    const guard = useGuard ? new ThinkingGuard(n, options.thinkingBudget ?? 1536) : null;
    const tGen = performance.now();
    const res = await model.generate({
      input_ids: int64(ids, [1, n]),
      attention_mask: ones(n),
      past_key_values: cache,
      max_new_tokens: maxNew,
      do_sample: (options.temperature ?? 0) > 0,
      temperature: options.temperature ?? 0.1,
      top_k: options.topK ?? 50,
      top_p: options.topP ?? 1.0,
      repetition_penalty: options.repetitionPenalty ?? 1.1,
      stopping_criteria: guard ? [stopper, guard] : stopper,
      streamer,
      return_dict_in_generate: true,
    });
    const genMs = performance.now() - tGen;
    const seq = res.sequences.tolist()[0].map(Number);
    cached.tokens = seq.slice(0, seq.length - 1); // the cache holds all but the last sampled token
    let gen = seq.slice(n);
    const thinkingCut = guard?.reason ?? null;
    if (thinkingCut) gen = gen.slice(0, guard.cutAt);
    return {
      ...base,
      thinkingCut,
      text: tokenizer.decode(gen, { skip_special_tokens: true }),
      stopped: stopRequested,
      genTokens: gen.length,
      genMs,
      hitLimit: gen.length >= maxNew,
    };
  }

  return { load, generate, stop, reset };
}