LocalWebAgent / js /llm.js
ereniko's picture
v0.7.1: local models without f16 shaders (Linux), centered Meta logo
cef9371 unverified
Raw History Blame Contribute Delete
3.02 kB
// Main-thread client for the model worker.
import { MODEL_DEVICE, graphsOf, effectiveModel } from "./config.js";
export class LLM {
/** model: an entry of LOCAL_MODELS (config.js). */
constructor(model) {
this.model = model;
this.supportsThinking = model.thinking;
this.worker = new Worker(new URL("./llm-worker.js", import.meta.url), { type: "module" });
this.pending = new Map();
this.nextId = 1;
this.ready = false;
this.onProgress = null; // ({file, loaded, total, status})
this.onStatus = null; // (text)
this.onGpu = null; // (info)
this.worker.onmessage = ({ data }) => this.#onMessage(data);
this.worker.onerror = (e) => this.#fail(new Error(e.message || "Model worker crashed"));
}
load() {
this.loading ??= new Promise((resolve, reject) => {
this.pending.set("load", { resolve, reject });
// Resolved at load time: the build depends on the f16 check done at startup.
const m = effectiveModel(this.model);
const { repo, dtype, textConfig, chatTemplateFile, emptyInputs, fullPrefill } = m;
this.worker.postMessage({
type: "load", modelId: repo, dtype, graphs: graphsOf(m), textConfig, chatTemplateFile, emptyInputs, fullPrefill, device: MODEL_DEVICE,
});
});
return this.loading;
}
/** Streams tokens to onEvent({type:'token'|'prefill', ...}); resolves with stats + full text. */
generate(messages, tools, options, onEvent) {
const id = this.nextId++;
return new Promise((resolve, reject) => {
this.pending.set(id, { resolve, reject, onEvent });
this.worker.postMessage({ type: "generate", id, messages, tools, options });
});
}
stop() { this.worker.postMessage({ type: "stop" }); }
/** Ends the worker, releasing the model's GPU memory. */
dispose() {
this.worker.terminate();
this.#fail(new Error("The local model was unloaded."));
}
resetCache() { this.worker.postMessage({ type: "reset" }); }
#fail(err) {
for (const p of this.pending.values()) p.reject(err);
this.pending.clear();
}
#onMessage(m) {
switch (m.type) {
case "progress": this.onProgress?.(m); break;
case "status": this.onStatus?.(m.text); break;
case "gpu": this.onGpu?.(m); break;
case "ready":
this.ready = true;
this.pending.get("load")?.resolve();
this.pending.delete("load");
break;
case "token":
case "prefill":
this.pending.get(m.id)?.onEvent?.(m);
break;
case "done":
this.pending.get(m.id)?.resolve(m);
this.pending.delete(m.id);
break;
case "error": {
const err = new Error(m.message);
const key = m.id ?? "load";
const p = this.pending.get(key) ?? this.pending.get("load");
if (p) { p.reject(err); this.pending.delete(key); }
else console.error("model worker:", m.message, m.stack);
if (!this.ready) this.loading = null; // allow retry
break;
}
}
}
}