ereniko commited on
Commit
f4e16d6
·
unverified ·
1 Parent(s): a8f5a2f

v0.3.1: lower peak memory while loading the model

Browse files

Weights are streamed into Cache Storage and read from disk tensor by tensor
during the GPU upload; phones load the model before booting the VM.
Source: github.com/erinoooo/LocalWebAgent@c0f9d66

Files changed (6) hide show
  1. js/config.js +1 -1
  2. js/lazy-weights.js +83 -0
  3. js/llm-engine.js +4 -2
  4. js/llm-worker.js +16 -5
  5. js/main.js +19 -4
  6. sw.js +2 -2
js/config.js CHANGED
@@ -1,5 +1,5 @@
1
  // Central configuration. Everything that pins a version or a location lives here.
2
- export const APP_VERSION = "0.3.0";
3
 
4
  // --- Language model (loaded from the Hugging Face Hub, cached by transformers.js) ---
5
  export const MODEL_ID = "LiquidAI/LFM3-1.7B-ONNX";
 
1
  // Central configuration. Everything that pins a version or a location lives here.
2
+ export const APP_VERSION = "0.3.1";
3
 
4
  // --- Language model (loaded from the Hugging Face Hub, cached by transformers.js) ---
5
  export const MODEL_ID = "LiquidAI/LFM3-1.7B-ONNX";
js/lazy-weights.js ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Low-memory loading of the model's external weights (model_<dtype>.onnx_data, ~1.4 GB).
2
+ //
3
+ // By default transformers.js reads the whole file into one Uint8Array and hands it to
4
+ // onnxruntime, which then copies it tensor by tensor into GPU buffers. Until that finishes
5
+ // the weights exist twice: once in JS memory and once on the GPU. On phones and other
6
+ // devices with shared CPU/GPU memory that peak is what gets the tab's GPU process killed.
7
+ //
8
+ // Instead we keep the file in Cache Storage (streamed there, never fully in memory) and
9
+ // give onnxruntime a Blob-backed stand-in. onnxruntime only ever calls .byteLength and
10
+ // .subarray(begin, end) on mounted external data, so each tensor is read from disk with
11
+ // FileReaderSync right before it is uploaded. Peak extra memory becomes one tensor.
12
+ //
13
+ // Worker-only (FileReaderSync). Callers fall back to the default path if this throws.
14
+
15
+ const CACHE_NAME = "transformers-cache"; // the cache transformers.js uses; setup.js checks it too
16
+
17
+ /** A Uint8Array (so transformers.js passes it through untouched) whose bytes live in a Blob. */
18
+ class BlobBytes extends Uint8Array {
19
+ #blob;
20
+ constructor(blob) {
21
+ super(0);
22
+ this.#blob = blob;
23
+ }
24
+ get byteLength() { return this.#blob.size; }
25
+ get length() { return this.#blob.size; }
26
+ subarray(begin = 0, end = this.#blob.size) {
27
+ return new Uint8Array(new FileReaderSync().readAsArrayBuffer(this.#blob.slice(begin, end)));
28
+ }
29
+ }
30
+
31
+ /** Streams url into Cache Storage, reporting progress like transformers.js does. */
32
+ async function download(cache, url, file, progress) {
33
+ const res = await fetch(url);
34
+ if (!res.ok || !res.body) throw new Error(`Download failed (${res.status}) for ${file}`);
35
+ const total = Number(res.headers.get("Content-Length")) || 0;
36
+ let loaded = 0;
37
+ let last = 0;
38
+ progress?.({ status: "initiate", file, loaded: 0, total });
39
+ const counted = res.body.pipeThrough(new TransformStream({
40
+ transform(chunk, ctl) {
41
+ loaded += chunk.byteLength;
42
+ const now = performance.now();
43
+ if (now - last > 200) { last = now; progress?.({ status: "progress", file, loaded, total }); }
44
+ ctl.enqueue(chunk);
45
+ },
46
+ }));
47
+ const headers = new Headers(res.headers);
48
+ await cache.put(url, new Response(counted, { status: 200, headers }));
49
+ if (total && loaded !== total) {
50
+ await cache.delete(url);
51
+ throw new Error(`Incomplete download of ${file} (${loaded} of ${total} bytes)`);
52
+ }
53
+ progress?.({ status: "done", file, loaded, total: total || loaded });
54
+ }
55
+
56
+ /**
57
+ * Returns from_pretrained options that make onnxruntime read the external weights
58
+ * lazily from Cache Storage, downloading them first if needed.
59
+ */
60
+ export async function lazyWeightsOptions(modelId, dtype, progress) {
61
+ if (typeof FileReaderSync === "undefined" || !self.caches) throw new Error("lazy weights unsupported here");
62
+ const name = `model_${dtype}.onnx_data`;
63
+ const file = `onnx/${name}`;
64
+ const url = `https://huggingface.co/${modelId}/resolve/main/${file}`;
65
+ const cache = await caches.open(CACHE_NAME);
66
+ let res = await cache.match(url);
67
+ if (!res) {
68
+ await download(cache, url, file, progress);
69
+ res = await cache.match(url);
70
+ if (!res) throw new Error("weights were not stored in Cache Storage");
71
+ } else {
72
+ const size = Number(res.headers.get("Content-Length")) || 0;
73
+ progress?.({ status: "done", file, loaded: size, total: size });
74
+ }
75
+ const blob = await res.blob(); // disk-backed in Chromium; not read into memory
76
+ if (!blob.size) throw new Error("cached weights are empty");
77
+ return {
78
+ // Stop transformers.js from fetching the .onnx_data itself...
79
+ use_external_data_format: false,
80
+ // ...and mount ours under the name the ONNX graph refers to.
81
+ session_options: { externalData: [{ path: name, data: new BlobBytes(blob) }] },
82
+ };
83
+ }
js/llm-engine.js CHANGED
@@ -64,10 +64,12 @@ export function createEngine(tf) {
64
  const int64 = (arr, dims) => new Tensor("int64", BigInt64Array.from(arr, BigInt), dims);
65
  const ones = (n) => new Tensor("int64", new BigInt64Array(n).fill(1n), [1, n]);
66
 
67
- async function load(modelId, { dtype, device, progress_callback, onStatus } = {}) {
 
 
68
  tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback });
69
  onStatus?.("Downloading / loading model weights");
70
- model = await AutoModelForCausalLM.from_pretrained(modelId, { dtype, device, progress_callback });
71
  const te = tokenizer.encode("</think>", { add_special_tokens: false });
72
  thinkEndId = te.length === 1 ? te[0] : -1;
73
  // The LFM3 ONNX graph takes `logits_to_keep` (rank-1 int64). transformers.js only
 
64
  const int64 = (arr, dims) => new Tensor("int64", BigInt64Array.from(arr, BigInt), dims);
65
  const ones = (n) => new Tensor("int64", new BigInt64Array(n).fill(1n), [1, n]);
66
 
67
+ // modelOptions: extra from_pretrained options (the browser worker uses it to hand
68
+ // onnxruntime the weights as a lazily read, disk-backed buffer).
69
+ async function load(modelId, { dtype, device, progress_callback, onStatus, modelOptions } = {}) {
70
  tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback });
71
  onStatus?.("Downloading / loading model weights");
72
+ model = await AutoModelForCausalLM.from_pretrained(modelId, { dtype, device, progress_callback, ...modelOptions });
73
  const te = tokenizer.encode("</think>", { add_special_tokens: false });
74
  thinkEndId = te.length === 1 ? te[0] : -1;
75
  // The LFM3 ONNX graph takes `logits_to_keep` (rank-1 int64). transformers.js only
js/llm-worker.js CHANGED
@@ -2,6 +2,7 @@
2
  // The generation logic lives in llm-engine.js (shared with the Node test harness).
3
  import * as tf from "../vendor/transformers.min.js";
4
  import { createEngine } from "./llm-engine.js";
 
5
 
6
  tf.env.allowLocalModels = false;
7
  tf.env.backends.onnx.wasm.wasmPaths = {
@@ -36,14 +37,24 @@ async function load({ modelId, dtype, device }) {
36
  const info = adapter.info ?? {};
37
  post({ type: "gpu", vendor: info.vendor, architecture: info.architecture, description: info.description });
38
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  await engine.load(modelId, {
40
  dtype,
41
  device,
42
- progress_callback: (p) => {
43
- if (p.status === "progress" || p.status === "done" || p.status === "initiate") {
44
- post({ type: "progress", file: p.file, loaded: p.loaded ?? 0, total: p.total ?? 0, status: p.status });
45
- }
46
- },
47
  onStatus: (text) => post({ type: "status", text }),
48
  });
49
  post({ type: "ready" });
 
2
  // The generation logic lives in llm-engine.js (shared with the Node test harness).
3
  import * as tf from "../vendor/transformers.min.js";
4
  import { createEngine } from "./llm-engine.js";
5
+ import { lazyWeightsOptions } from "./lazy-weights.js";
6
 
7
  tf.env.allowLocalModels = false;
8
  tf.env.backends.onnx.wasm.wasmPaths = {
 
37
  const info = adapter.info ?? {};
38
  post({ type: "gpu", vendor: info.vendor, architecture: info.architecture, description: info.description });
39
  }
40
+ const progress_callback = (p) => {
41
+ if (p.status === "progress" || p.status === "done" || p.status === "initiate") {
42
+ post({ type: "progress", file: p.file, loaded: p.loaded ?? 0, total: p.total ?? 0, status: p.status });
43
+ }
44
+ };
45
+ // Read the ~1.4 GB of weights from disk tensor by tensor instead of holding them all
46
+ // in memory while they are copied to the GPU (see lazy-weights.js).
47
+ let modelOptions;
48
+ try {
49
+ modelOptions = await lazyWeightsOptions(modelId, dtype, progress_callback);
50
+ } catch (err) {
51
+ console.warn("Low-memory weight loading unavailable, using the default path:", err);
52
+ }
53
  await engine.load(modelId, {
54
  dtype,
55
  device,
56
+ progress_callback,
57
+ modelOptions,
 
 
 
58
  onStatus: (text) => post({ type: "status", text }),
59
  });
60
  post({ type: "ready" });
js/main.js CHANGED
@@ -208,7 +208,7 @@ async function start() {
208
  agent = new Agent({ llm, vm, settings, ui: chatUI });
209
  setComposerEnabled();
210
 
211
- const vmTask = (async () => {
212
  status("vm", "busy", "Booting");
213
  try {
214
  diskManifest ??= await setup.installedDisk();
@@ -223,15 +223,30 @@ async function start() {
223
  vm.term?.writeln("\r\n\x1b[31mFailed to start Linux: " + err.message + "\x1b[0m");
224
  }
225
  setComposerEnabled();
226
- })();
227
- const modelTask = llm.ready ? Promise.resolve() : loadModel().catch((err) => {
228
  console.error(err);
229
  addNotice("The model failed to load: " + err.message, "error");
230
  });
231
- await Promise.all([vmTask, modelTask]);
 
 
 
 
 
 
 
 
232
  setComposerEnabled();
233
  }
234
 
 
 
 
 
 
 
 
235
  // ---------- conversation rendering ----------
236
  const thread = () => $("#messages");
237
  const scroller = () => $("#scroller");
 
208
  agent = new Agent({ llm, vm, settings, ui: chatUI });
209
  setComposerEnabled();
210
 
211
+ const bootVm = async () => {
212
  status("vm", "busy", "Booting");
213
  try {
214
  diskManifest ??= await setup.installedDisk();
 
223
  vm.term?.writeln("\r\n\x1b[31mFailed to start Linux: " + err.message + "\x1b[0m");
224
  }
225
  setComposerEnabled();
226
+ };
227
+ const loadModelOnce = () => llm.ready ? Promise.resolve() : loadModel().catch((err) => {
228
  console.error(err);
229
  addNotice("The model failed to load: " + err.message, "error");
230
  });
231
+ if (memoryConstrained()) {
232
+ // Upload the weights to the GPU before Linux takes its share of memory: on phones
233
+ // the two together at peak are what makes the browser kill the GPU process.
234
+ status("vm", "busy", "Waiting for the model");
235
+ await loadModelOnce();
236
+ await bootVm();
237
+ } else {
238
+ await Promise.all([bootVm(), loadModelOnce()]);
239
+ }
240
  setComposerEnabled();
241
  }
242
 
243
+ // Phones, tablets and machines reporting under 8 GB of RAM (Chrome caps deviceMemory at 8).
244
+ function memoryConstrained() {
245
+ if (navigator.userAgentData?.mobile) return true;
246
+ if (/Android|iPhone|iPad|Mobile/i.test(navigator.userAgent)) return true;
247
+ return (navigator.deviceMemory ?? 8) < 8;
248
+ }
249
+
250
  // ---------- conversation rendering ----------
251
  const thread = () => $("#messages");
252
  const scroller = () => $("#scroller");
sw.js CHANGED
@@ -6,7 +6,7 @@
6
  // (__vm__/disk.ext2), backed by the chunks the setup screen stored in Cache Storage.
7
  // 4. Adds the cross-origin isolation headers (COOP/COEP) that SharedArrayBuffer,
8
  // CheerpX and multi-threaded WASM need, even when a host forgets to send them.
9
- const SHELL_VERSION = "lwa-shell-0.3.0";
10
  const CX_ORIGIN = "https://cxrtnc.leaningtech.com";
11
  const CX_CACHE_PREFIX = "lwa-cheerpx-";
12
  const DISK_CACHE = "lwa-disk";
@@ -18,7 +18,7 @@ const chunkKey = (i) => SCOPE + "__vm__/chunk/" + i;
18
  const SHELL_FILES = [
19
  "./", "index.html", "css/app.css", "icon.svg",
20
  "js/config.js", "js/config-defaults.js", "js/main.js", "js/setup.js", "js/vm.js", "js/agent.js", "js/llm.js",
21
- "js/llm-worker.js", "js/llm-engine.js", "js/prompts.js", "js/toolcalls.js", "js/markdown.js", "js/files.js",
22
  "vendor/transformers.min.js", "vendor/highlight.min.js",
23
  "vendor/ort/ort-wasm-simd-threaded.asyncify.mjs", "vendor/ort/ort-wasm-simd-threaded.asyncify.wasm",
24
  "vendor/xterm/xterm.mjs", "vendor/xterm/xterm.css", "vendor/xterm/addon-fit.mjs",
 
6
  // (__vm__/disk.ext2), backed by the chunks the setup screen stored in Cache Storage.
7
  // 4. Adds the cross-origin isolation headers (COOP/COEP) that SharedArrayBuffer,
8
  // CheerpX and multi-threaded WASM need, even when a host forgets to send them.
9
+ const SHELL_VERSION = "lwa-shell-0.3.1";
10
  const CX_ORIGIN = "https://cxrtnc.leaningtech.com";
11
  const CX_CACHE_PREFIX = "lwa-cheerpx-";
12
  const DISK_CACHE = "lwa-disk";
 
18
  const SHELL_FILES = [
19
  "./", "index.html", "css/app.css", "icon.svg",
20
  "js/config.js", "js/config-defaults.js", "js/main.js", "js/setup.js", "js/vm.js", "js/agent.js", "js/llm.js",
21
+ "js/llm-worker.js", "js/llm-engine.js", "js/lazy-weights.js", "js/prompts.js", "js/toolcalls.js", "js/markdown.js", "js/files.js",
22
  "vendor/transformers.min.js", "vendor/highlight.min.js",
23
  "vendor/ort/ort-wasm-simd-threaded.asyncify.mjs", "vendor/ort/ort-wasm-simd-threaded.asyncify.wasm",
24
  "vendor/xterm/xterm.mjs", "vendor/xterm/xterm.css", "vendor/xterm/addon-fit.mjs",