// Main-thread client for the model worker. import { MODEL_DEVICE, graphsOf, effectiveModel } from "./config.js"; export class LLM { /** model: an entry of LOCAL_MODELS (config.js). */ constructor(model) { this.model = model; this.supportsThinking = model.thinking; this.worker = new Worker(new URL("./llm-worker.js", import.meta.url), { type: "module" }); this.pending = new Map(); this.nextId = 1; this.ready = false; this.onProgress = null; // ({file, loaded, total, status}) this.onStatus = null; // (text) this.onGpu = null; // (info) this.worker.onmessage = ({ data }) => this.#onMessage(data); this.worker.onerror = (e) => this.#fail(new Error(e.message || "Model worker crashed")); } load() { this.loading ??= new Promise((resolve, reject) => { this.pending.set("load", { resolve, reject }); // Resolved at load time: the build depends on the f16 check done at startup. const m = effectiveModel(this.model); const { repo, dtype, textConfig, chatTemplateFile, emptyInputs, fullPrefill } = m; this.worker.postMessage({ type: "load", modelId: repo, dtype, graphs: graphsOf(m), textConfig, chatTemplateFile, emptyInputs, fullPrefill, device: MODEL_DEVICE, }); }); return this.loading; } /** Streams tokens to onEvent({type:'token'|'prefill', ...}); resolves with stats + full text. */ generate(messages, tools, options, onEvent) { const id = this.nextId++; return new Promise((resolve, reject) => { this.pending.set(id, { resolve, reject, onEvent }); this.worker.postMessage({ type: "generate", id, messages, tools, options }); }); } stop() { this.worker.postMessage({ type: "stop" }); } /** Ends the worker, releasing the model's GPU memory. */ dispose() { this.worker.terminate(); this.#fail(new Error("The local model was unloaded.")); } resetCache() { this.worker.postMessage({ type: "reset" }); } #fail(err) { for (const p of this.pending.values()) p.reject(err); this.pending.clear(); } #onMessage(m) { switch (m.type) { case "progress": this.onProgress?.(m); break; case "status": this.onStatus?.(m.text); break; case "gpu": this.onGpu?.(m); break; case "ready": this.ready = true; this.pending.get("load")?.resolve(); this.pending.delete("load"); break; case "token": case "prefill": this.pending.get(m.id)?.onEvent?.(m); break; case "done": this.pending.get(m.id)?.resolve(m); this.pending.delete(m.id); break; case "error": { const err = new Error(m.message); const key = m.id ?? "load"; const p = this.pending.get(key) ?? this.pending.get("load"); if (p) { p.reject(err); this.pending.delete(key); } else console.error("model worker:", m.message, m.stack); if (!this.ready) this.loading = null; // allow retry break; } } } }