// Generated by scripts/build_js_bridge.mjs from js/src/llama_webgpu_bridge.js. // Do not edit this file directly; update js/src/ and run npm run build:js. // js/src/internal/abort.ts function createAbortError(message) { if (typeof DOMException === "function") { return new DOMException(message, "AbortError"); } const error = new Error(message); error.name = "AbortError"; return error; } function throwIfAborted(signal, message = "Bridge operation was cancelled.") { if (signal?.aborted) { throw createAbortError(message); } } // js/src/internal/embedding.ts var EmbeddingError = class extends Error { constructor(code, message) { super(message); this.name = "EmbeddingError"; this.code = code; } }; var EMPTY_EMBEDDING_CAPABILITIES = { apiVersion: 1, text: false, image: false, audio: false }; var EMBEDDING_MEDIA_MARKER = "<__media__>"; function embeddingAssetVersion(...sources) { for (const source of [...sources, import.meta.url]) { try { const version = new URL(source || "", import.meta.url).pathname.match(/(?:^|[/@])(v?\d+\.\d+\.\d+(?:-\d+)?)(?:[/]|$)/)?.[1]; if (version) return version; } catch (_) { } } return "unversioned/local"; } function missingEmbeddingCapability(version = embeddingAssetVersion()) { return new EmbeddingError( "EMBEDDING_CAPABILITY_UNAVAILABLE", `Loaded bridge/assets version ${version} has no known media-embedding capability API version 1. Upgrade the loaded bridge assets.` ); } function embeddingCapabilitiesFrom(value, version = embeddingAssetVersion()) { let parsed = value; if (typeof value === "string") { try { parsed = JSON.parse(value); } catch (_) { throw missingEmbeddingCapability(version); } } const caps = parsed; if (!caps || caps.apiVersion !== 1 || typeof caps.text !== "boolean" || typeof caps.image !== "boolean" || typeof caps.audio !== "boolean") { throw missingEmbeddingCapability(version); } return { apiVersion: 1, text: caps.text, image: caps.image, audio: caps.audio }; } function snapshotEmbedInput(input) { if (!Array.isArray(input)) return String(input); if (!Array.isArray(input) || input.length === 0) { throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding parts must be a nonempty array."); } const parts = Array.from(input, (part) => { if (!part || typeof part !== "object") { throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Invalid embedding part."); } if (part.type === "text") { if (typeof part.text !== "string") { throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding text part must contain a string."); } return { type: "text", text: part.text }; } if (part.type !== "image" && part.type !== "audio") { throw new EmbeddingError("EMBEDDING_MODALITY_UNSUPPORTED", "Embedding part modality is unsupported."); } if ("path" in part) { throw new EmbeddingError("EMBEDDING_MODALITY_UNSUPPORTED", "Browser embeddings require media bytes, URL or PCM; filesystem paths are unsupported."); } const sourceCount = Number(part.bytes != null) + Number(part.url != null) + Number("samples" in part && part.samples != null); if (sourceCount !== 1 || part.bytes != null && (!(part.bytes instanceof Uint8Array) || part.bytes.length === 0) || part.url != null && (typeof part.url !== "string" || part.url.length === 0)) { throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding media requires exactly one nonempty supported source."); } if ("samples" in part && part.samples != null) { if (part.type !== "audio" || !(part.samples instanceof Float32Array) || part.samples.length === 0 || !part.samples.every(Number.isFinite)) { throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding PCM must be finite mono 16 kHz Float32 samples."); } } return { ...part }; }); const merged = []; for (const part of parts) { const previous = merged.at(-1); if (part.type === "text" && previous?.type === "text") previous.text += part.text; else merged.push(part); } if (merged.some((part) => part.type === "text" && part.text.includes(EMBEDDING_MEDIA_MARKER))) { throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Reserved media marker in embedding text."); } return merged; } function requireEmbeddingModalities(input, caps, hasProjector) { if (typeof input === "string") return; if (!caps.text) { throw new EmbeddingError("MODEL_NOT_EMBEDDING", "Loaded model does not support pooled embeddings."); } for (const part of input) { if (part.type === "text") continue; if (!hasProjector) { throw new EmbeddingError("EMBEDDING_PROJECTOR_MISSING", "Media embeddings require a loaded projector."); } if (!caps[part.type]) { throw new EmbeddingError("EMBEDDING_MODALITY_UNSUPPORTED", "Loaded projector/model does not support the requested embedding modality."); } } } function embeddingCoreError(rc, version = embeddingAssetVersion()) { if (rc === -20) return missingEmbeddingCapability(version); const errors = { [-21]: ["MODEL_NOT_EMBEDDING", "Loaded model does not support this pooled embedding operation."], [-22]: ["EMBEDDING_PROJECTOR_MISSING", "Media embeddings require a loaded projector."], [-23]: ["EMBEDDING_INPUT_TOO_LARGE", "Expanded embedding input exceeds the allocated context, batch, sequence or single microbatch capacity."], [-24]: ["EMBEDDING_MODALITY_UNSUPPORTED", "Loaded projector does not support the requested embedding modality or PCM rate."] }; const [code, message] = errors[rc] || ["EMBEDDING_FAILED", "Embedding execution failed."]; return new EmbeddingError(code, message); } // js/src/internal/completion_options.ts var SPECULATIVE_DECODING_STRATEGIES = Object.freeze([ "draft-simple", "draft-eagle3", "draft-mtp", "draft-dflash", "draft-dspark", "ngram-simple", "ngram-map-k", "ngram-map-k4v", "ngram-mod", "ngram-cache" ]); function speculativeFlags(isSupported) { return Object.fromEntries( SPECULATIVE_DECODING_STRATEGIES.map((strategy) => [strategy, isSupported(strategy)]) ); } var NO_COMPLETION_CAPABILITIES = Object.freeze({ presencePenalty: false, minP: false, thinkingBudget: false, speculativeDecoding: Object.freeze(speculativeFlags(() => false)) }); var MAX_NGRAM_SIZE = 65535; var MAX_INT32 = 2147483647; var MAX_THINKING_BUDGET_TOKENS = 2147483647; function optionalNumber(value, name, isValid, range) { if (value == null) { return null; } if (typeof value !== "number" || !Number.isFinite(Math.fround(value)) || !isValid(value)) { throw new RangeError(`CompletionOptions.${name} must be a finite number${range}; got ${String(value)}.`); } return value; } function tag(budget, name) { const value = budget[name]; if (typeof value !== "string" || value.trim().length === 0) { throw new TypeError(`CompletionOptions.thinkingBudget.${name} must be a non-empty string.`); } return value; } function thinkingBudget(value, hasMediaParts) { if (value == null) { return null; } if (typeof value !== "object") { throw new TypeError("CompletionOptions.thinkingBudget must be an object."); } const budget = value; const maxTokens = budget.maxTokens; if (!Number.isInteger(maxTokens) || maxTokens < 0 || maxTokens > MAX_THINKING_BUDGET_TOKENS) { throw new RangeError( `CompletionOptions.thinkingBudget.maxTokens must be an integer from 0 to ${MAX_THINKING_BUDGET_TOKENS}; got ${String(maxTokens)}.` ); } const startTag = tag(budget, "startTag"); const endTag = tag(budget, "endTag"); const forcedMessage = budget.forcedMessage ?? ""; if (typeof forcedMessage !== "string") { throw new TypeError("CompletionOptions.thinkingBudget.forcedMessage must be a string."); } if (hasMediaParts) { throw new Error("CompletionOptions.thinkingBudget supports text-only prompts; remove the media parts."); } return { maxTokens, startTag, endTag, forcedMessage }; } function speculativeInteger(config, name, min, max) { const value = config[name]; if (value == null) { return null; } if (!Number.isInteger(value) || value < min || value > max) { throw new RangeError( `CompletionOptions.speculativeDecoding.${name} must be an integer from ${min} to ${max}; got ${String(value)}.` ); } return value; } function speculativeProbability(config, name) { const value = config[name]; if (value == null) { return null; } if (typeof value !== "number" || !(value >= 0 && value <= 1)) { throw new RangeError( `CompletionOptions.speculativeDecoding.${name} must be a number from 0 to 1; got ${String(value)}.` ); } return value; } function ngramCacheSource(config, name) { const value = config[name]; if (value == null) { return null; } const isBytes = value instanceof ArrayBuffer || ArrayBuffer.isView(value); if (!(typeof value === "string" || isBytes)) { throw new TypeError( `CompletionOptions.speculativeDecoding.${name} must be a URL string, an ArrayBuffer or a typed array.` ); } if ((typeof value === "string" ? value.length : value.byteLength) === 0) { throw new TypeError(`CompletionOptions.speculativeDecoding.${name} is empty.`); } return value; } function isDraftStrategy(strategy) { return strategy.startsWith("draft-"); } function speculativeDecoding(value, conflicts) { if (value == null) { return null; } if (typeof value !== "object") { throw new TypeError("CompletionOptions.speculativeDecoding must be an object."); } const config = value; const requested = config.strategies; if (!Array.isArray(requested) || requested.length === 0) { throw new TypeError("CompletionOptions.speculativeDecoding.strategies must be a non-empty array."); } const strategies = []; for (const strategy of requested) { if (!SPECULATIVE_DECODING_STRATEGIES.includes(strategy)) { throw new TypeError( `CompletionOptions.speculativeDecoding.strategies has an unknown strategy ${JSON.stringify(strategy)}; expected one of ${SPECULATIVE_DECODING_STRATEGIES.join(", ")}.` ); } if (!strategies.includes(strategy)) { strategies.push(strategy); } } const draftTokenMax = speculativeInteger(config, "draftTokenMax", 0, MAX_INT32); const draftTokenMin = speculativeInteger(config, "draftTokenMin", 0, MAX_INT32); const minProbability = speculativeProbability(config, "minProbability"); const draftSplitProbability = speculativeProbability(config, "draftSplitProbability"); const ngramSizeN = speculativeInteger(config, "ngramSizeN", 1, MAX_NGRAM_SIZE); const ngramSizeM = speculativeInteger(config, "ngramSizeM", 1, MAX_NGRAM_SIZE); const ngramMinHits = speculativeInteger(config, "ngramMinHits", 1, MAX_NGRAM_SIZE); const ngramMatch = speculativeInteger(config, "ngramMatch", 1, MAX_NGRAM_SIZE); const ngramTokenMin = speculativeInteger(config, "ngramTokenMin", 0, MAX_INT32); const ngramTokenMax = speculativeInteger(config, "ngramTokenMax", 0, MAX_INT32); const ngramCacheStatic = ngramCacheSource(config, "ngramCacheStatic"); const ngramCacheDynamic = ngramCacheSource(config, "ngramCacheDynamic"); if (strategies.filter(isDraftStrategy).length > 1) { throw new Error( "CompletionOptions.speculativeDecoding can mix n-gram strategies with at most one draft-* strategy." ); } if (!strategies.some(isDraftStrategy) && (draftTokenMin != null || minProbability != null || draftSplitProbability != null)) { throw new Error( "CompletionOptions.speculativeDecoding: n-gram strategies use token history and do not support draftTokenMin, minProbability or draftSplitProbability unless a draft-* strategy is also enabled." ); } if ((ngramCacheStatic != null || ngramCacheDynamic != null) && !strategies.includes("ngram-cache")) { throw new Error( "CompletionOptions.speculativeDecoding: ngramCacheStatic and ngramCacheDynamic need the ngram-cache strategy." ); } if (ngramTokenMin != null && ngramTokenMax != null && ngramTokenMin > ngramTokenMax) { throw new RangeError( `CompletionOptions.speculativeDecoding.ngramTokenMin (${ngramTokenMin}) must not exceed ngramTokenMax (${ngramTokenMax}).` ); } let resolvedDraftTokenMax = 0; for (const strategy of strategies) { let strategyMax; if (isDraftStrategy(strategy)) { strategyMax = draftTokenMax ?? 3; } else if (strategy === "ngram-mod") { strategyMax = ngramTokenMax ?? draftTokenMax ?? 64; } else if (strategy === "ngram-cache") { strategyMax = draftTokenMax ?? 8; } else { strategyMax = ngramSizeM ?? 48; } resolvedDraftTokenMax = Math.max(resolvedDraftTokenMax, strategyMax); } if (resolvedDraftTokenMax === 0) { resolvedDraftTokenMax = 64; } if ((draftTokenMin ?? 0) > resolvedDraftTokenMax) { throw new RangeError( `CompletionOptions.speculativeDecoding.draftTokenMin (${draftTokenMin}) must not exceed the draft token maximum (${resolvedDraftTokenMax}).` ); } if (conflicts.hasMediaParts) { throw new Error("CompletionOptions.speculativeDecoding supports text-only prompts; remove the media parts."); } if (conflicts.hasThinkingBudget) { throw new Error("CompletionOptions.speculativeDecoding cannot be combined with CompletionOptions.thinkingBudget."); } if (conflicts.hasGrammar) { throw new Error("CompletionOptions.speculativeDecoding does not support CompletionOptions.grammar."); } return { strategies, draftTokenMax: resolvedDraftTokenMax, draftTokenMin: draftTokenMin ?? 0, minProbability: minProbability ?? -1, draftSplitProbability: draftSplitProbability ?? -1, ngramSizeN: ngramSizeN ?? 0, ngramSizeM: ngramSizeM ?? 0, ngramMinHits: ngramMinHits ?? 0, ngramMatch: ngramMatch ?? 0, ngramTokenMin: ngramTokenMin ?? -1, ngramTokenMax: ngramTokenMax ?? (strategies.includes("ngram-mod") ? draftTokenMax ?? 0 : 0), ngramCacheStatic, ngramCacheDynamic }; } function resolveCompletionSamplingOptions(options) { const hasMediaParts = Array.isArray(options?.parts) && options.parts.length > 0; const budget = thinkingBudget(options?.thinkingBudget, hasMediaParts); return { minP: optionalNumber(options?.minP, "minP", (value) => value >= 0 && value <= 1, " from 0 to 1") ?? 0, presencePenalty: optionalNumber(options?.presencePenalty, "presencePenalty", () => true, "") ?? 0, thinkingBudget: budget, speculative: speculativeDecoding(options?.speculativeDecoding, { hasMediaParts, hasGrammar: typeof options?.grammar === "string" && options.grammar.length > 0, hasThinkingBudget: budget != null }) }; } function completionCapabilitiesFrom(raw) { let parsed = null; try { parsed = JSON.parse(raw || "{}"); } catch (_) { return { ...NO_COMPLETION_CAPABILITIES }; } const speculative = parsed?.speculativeDecoding; return { presencePenalty: parsed?.presencePenalty === true, minP: parsed?.minP === true, thinkingBudget: parsed?.thinkingBudget === true, speculativeDecoding: speculativeFlags((strategy) => speculative?.[strategy] === true) }; } function requireCompletionCapabilities(sampling, capabilities) { const missing = [ sampling.minP !== 0 && !capabilities.minP ? "minP" : null, sampling.presencePenalty !== 0 && !capabilities.presencePenalty ? "presencePenalty" : null, sampling.thinkingBudget != null && !capabilities.thinkingBudget ? "thinkingBudget" : null, sampling.speculative != null && !Object.values(capabilities.speculativeDecoding).includes(true) ? "speculativeDecoding" : null ].filter((name) => name != null); if (missing.length > 0) { throw new Error( `The loaded WebGPU core does not support CompletionOptions.${missing.join(" or CompletionOptions.")}.` ); } } // js/src/internal/constants.ts var defaultModelCacheName = "llamadart-webgpu-model-cache-v1"; var BRIDGE_DISPOSED_MESSAGE = "Bridge has been disposed."; var GENERATION_ALREADY_ACTIVE_RC = -7; var GENERATION_ALREADY_ACTIVE_MESSAGE = "Generation is already active on this bridge runtime."; var INVALID_GRAMMAR_ERROR_TEXT = "Failed to initialize sampler chain (invalid grammar)"; // js/src/internal/typed_values.ts function toUint8Array(value) { if (!value) { return null; } if (value instanceof Uint8Array) { return value; } if (ArrayBuffer.isView(value)) { return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); } if (value instanceof ArrayBuffer) { return new Uint8Array(value); } if (Array.isArray(value)) { return Uint8Array.from(value.map((v) => Number(v) & 255)); } return null; } function bufferSourceBytes(source) { if (source instanceof ArrayBuffer) { return new Uint8Array(source); } if (ArrayBuffer.isView(source)) { return new Uint8Array(source.buffer, source.byteOffset, source.byteLength); } return null; } function toFloat32Array(value) { if (!value) { return null; } if (value instanceof Float32Array) { return value; } if (ArrayBuffer.isView(value)) { return new Float32Array( value.buffer, value.byteOffset, Math.floor(value.byteLength / Float32Array.BYTES_PER_ELEMENT) ); } if (value instanceof ArrayBuffer) { return new Float32Array(value); } if (Array.isArray(value)) { return Float32Array.from(value.map((v) => Number(v) || 0)); } return null; } function isInt32(value) { return typeof value === "number" && Number.isInteger(value) && value >= -2147483648 && value <= 2147483647; } // js/src/internal/decision.ts var DECISION_API_VERSION = 1; var DECISION_WORKER_TIMEOUT_PER_SEQUENCE_MS = 60 * 1e3; function decisionHandleFrom(handle) { if (!isInt32(handle) || handle <= 0) { throw new TypeError(`Decision head handle must be a positive integer, got ${String(handle)}.`); } return handle; } function decisionIntegerList(value, label) { const isList = Array.isArray(value) || ArrayBuffer.isView(value) && !(value instanceof DataView); if (!isList) { throw new TypeError(`${label} must be an array or typed array of integers.`); } const list = value; const out = new Int32Array(list.length); for (let index = 0; index < list.length; index += 1) { const item = list[index]; if (!isInt32(item)) { throw new TypeError(`${label}[${index}] is ${String(item)}; expected a 32-bit integer.`); } out[index] = item; } return out; } function normalizeDecisionSequences(sequences) { if (!Array.isArray(sequences)) { throw new TypeError("Decision sequences must be an array."); } return sequences.map((sequence, index) => { const label = `Decision sequence ${index}`; if (!sequence || typeof sequence !== "object") { throw new TypeError(`${label} must be an object with tokens, markers and questionType.`); } const tokens = decisionIntegerList(sequence.tokens, `${label} tokens`); const markers = decisionIntegerList(sequence.markers, `${label} markers`); const questionType = sequence.questionType; if (!isInt32(questionType)) { throw new TypeError( `${label} questionType is ${String(questionType)}; expected 0 (choice), 1 (score) or 2 (noul).` ); } return { tokens, markers, questionType }; }); } function encodeDecisionSequences(sequences) { const encoded = normalizeDecisionSequences(sequences); let words = 1; for (const sequence of encoded) { words += 3 + sequence.tokens.length + sequence.markers.length; } const bytes = new Uint8Array(words * 4); const view = new DataView(bytes.buffer); let offset = 0; const put = (value) => { view.setInt32(offset, value, true); offset += 4; }; put(encoded.length); for (const sequence of encoded) { put(sequence.questionType); put(sequence.tokens.length); put(sequence.markers.length); sequence.tokens.forEach(put); sequence.markers.forEach(put); } return bytes; } function decodeDecisionOutputs(bytes, count) { const malformed = () => new Error("Decision output from the WebGPU core is malformed."); const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); let offset = 0; const floats = (length) => { const values = new Float32Array(length); for (let index = 0; index < length; index += 1) { values[index] = view.getFloat32(offset, true); offset += 4; } return values; }; const outputs = []; for (let index = 0; index < count; index += 1) { if (bytes.byteLength - offset < 8) { throw malformed(); } const logitCount = view.getInt32(offset, true); const actCount = view.getInt32(offset + 4, true); offset += 8; if (logitCount < 0 || actCount < 0 || (logitCount + actCount) * 4 > bytes.byteLength - offset) { throw malformed(); } outputs.push({ logits: floats(logitCount), actLogits: floats(actCount) }); } if (offset !== bytes.byteLength) { throw malformed(); } return outputs; } // js/src/internal/logging.ts function logLevelForName(level) { switch (level) { case "debug": return 0; case "log": case "info": return 1; case "warn": return 2; case "error": return 3; default: return 1; } } function logThresholdForConfiguredLevel(level) { switch (level) { case 0: return 99; case 1: return 0; case 2: return 1; case 3: return 2; case 4: return 3; default: return 1; } } // js/src/internal/lora.ts var LORA_API_VERSION = 1; var LORA_LOAD_ABORT_MESSAGE = "LoRA adapter load was cancelled."; function loraHandleFrom(handle) { if (!isInt32(handle) || handle <= 0) { throw new TypeError(`LoRA adapter handle must be a positive integer, got ${String(handle)}.`); } return handle; } function loraScaleFrom(scale) { if (typeof scale !== "number" || !Number.isFinite(scale)) { throw new TypeError(`LoRA adapter scale must be a finite number, got ${String(scale)}.`); } if (!Number.isFinite(Math.fround(scale))) { throw new RangeError(`LoRA adapter scale ${String(scale)} is outside the 32-bit float range.`); } return scale; } function staleLoraAdapterError(handle) { return new Error( `LoRA adapter ${handle} is not loaded; its model was unloaded or replaced. Load the adapter again.` ); } // js/src/internal/model_source.ts function cloneModelSource(source) { if (Array.isArray(source)) { return source.slice(); } return String(source || "").trim(); } function hasModelSource(source) { return Array.isArray(source) ? source.length > 0 : typeof source === "string" && source.length > 0; } function basenameFromUrl(url) { try { const parsed = new URL(url, typeof window !== "undefined" ? window.location.href : void 0); const pathname = parsed.pathname || ""; const name = pathname.split("/").pop() || "model.gguf"; return name.includes("?") ? name.split("?")[0] : name; } catch (_) { const parts = String(url).split("/"); return parts[parts.length - 1] || "model.gguf"; } } function normalizeAbsoluteUrl(url) { try { return new URL(url, typeof window !== "undefined" ? window.location.href : void 0).toString(); } catch (_) { return String(url); } } function parseSplitShardPattern(fileName) { if (typeof fileName !== "string" || fileName.length === 0) { return null; } const match = fileName.match(/^(.*)-(\d{4,6})-of-(\d{4,6})\.gguf$/i); if (!match) { return null; } const total = Number(match[3]); if (!Number.isInteger(total) || total < 2 || total > 512) { return null; } return { prefix: match[1], width: Math.max(match[2].length, match[3].length), total }; } function expandModelShardUrls(modelUrlOrUrls) { if (Array.isArray(modelUrlOrUrls)) { return modelUrlOrUrls.map((value) => String(value || "").trim()).filter((value) => value.length > 0).map((value) => normalizeAbsoluteUrl(value)); } const source = String(modelUrlOrUrls || "").trim(); if (source.length === 0) { return []; } try { const parsed = new URL(source, typeof window !== "undefined" ? window.location.href : void 0); const pathname = parsed.pathname || ""; const slash = pathname.lastIndexOf("/"); const dirPath = slash >= 0 ? pathname.slice(0, slash + 1) : ""; const fileName = slash >= 0 ? pathname.slice(slash + 1) : pathname; const split = parseSplitShardPattern(fileName); if (!split) { return [parsed.toString()]; } const totalShardId = String(split.total).padStart(split.width, "0"); const urls = []; for (let shardIndex = 1; shardIndex <= split.total; shardIndex += 1) { const shardId = String(shardIndex).padStart(split.width, "0"); parsed.pathname = `${dirPath}${split.prefix}-${shardId}-of-${totalShardId}.gguf`; urls.push(parsed.toString()); } return urls; } catch (_) { return [source]; } } // js/src/internal/core_loader.ts function normalizeFactory(moduleExport) { if (typeof moduleExport === "function") { return moduleExport; } if (moduleExport && typeof moduleExport.default === "function") { return moduleExport.default; } if (moduleExport && typeof moduleExport.createLlamaWebGpuCoreModule === "function") { return moduleExport.createLlamaWebGpuCoreModule; } throw new Error("Unable to resolve llama_webgpu_core factory function"); } async function importCoreFactory(moduleUrl) { const exportedModule = await import(moduleUrl); return normalizeFactory(exportedModule); } // js/src/internal/parse.ts function parsePositiveInteger(value) { const numeric = Number(value); if (!Number.isFinite(numeric) || numeric <= 0) { return 0; } return Math.trunc(numeric); } function parseInteger(value, fallback = 0) { const numeric = Number(value); if (!Number.isFinite(numeric)) { return fallback; } return Math.trunc(numeric); } function parseBooleanFlag(value, fallback = false) { if (typeof value === "boolean") { return value; } if (typeof value === "number" && Number.isFinite(value)) { return value !== 0; } return fallback; } function parseOptionalBooleanFlag(value) { if (typeof value === "boolean") { return value ? 1 : 0; } if (typeof value === "number" && Number.isFinite(value)) { return value !== 0 ? 1 : 0; } return -1; } function parseEnumValue(value, allowed, fallback) { const parsed = parseInteger(value, fallback); return allowed.includes(parsed) ? parsed : fallback; } function parsePositiveNumber(value) { const numeric = Number(value); return Number.isFinite(numeric) && numeric > 0 ? numeric : 0; } // js/src/internal/download.ts function hasReadableResponseStream(response) { return !!(response && response.body && typeof response.body.getReader === "function"); } function sumProgressValues(values) { let total = 0; for (const value of values || []) { const numeric = Number(value); if (Number.isFinite(numeric) && numeric > 0) { total += numeric; } } return total; } function parseTotalFromContentRangeHeader(contentRangeHeader) { if (typeof contentRangeHeader !== "string" || contentRangeHeader.length === 0) { return 0; } const slash = contentRangeHeader.lastIndexOf("/"); if (slash < 0 || slash + 1 >= contentRangeHeader.length) { return 0; } return parsePositiveInteger(contentRangeHeader.slice(slash + 1)); } function inferResponseTotalBytes(response, loadedFallback = 0) { if (!response || !response.headers) { return parsePositiveInteger(loadedFallback); } const linkedSize = parsePositiveInteger(response.headers.get("x-linked-size")); if (linkedSize > 0) { return linkedSize; } const contentRangeTotal = parseTotalFromContentRangeHeader( response.headers.get("content-range") ); if (contentRangeTotal > 0) { return contentRangeTotal; } const contentLength = parsePositiveInteger(response.headers.get("content-length")); if (contentLength > 0) { return contentLength; } return parsePositiveInteger(loadedFallback); } function isRetryableStreamNetworkError(error) { const text = String(error || "").toLowerCase(); return text.includes("network error") || text.includes("failed to fetch") || text.includes("networkerror") || text.includes("err_network_io_suspended") || text.includes("the network connection was lost") || text.includes("connection reset") || text.includes("timeout") || text.includes("timed out"); } async function readStreamChunkWithTimeout(reader, timeoutMs, label = "stream read") { const resolvedTimeout = Number(timeoutMs); if (!Number.isFinite(resolvedTimeout) || resolvedTimeout <= 0) { return reader.read(); } let timeoutHandle = null; try { return await Promise.race([ reader.read(), new Promise((_, reject) => { timeoutHandle = globalThis.setTimeout(() => { reject(new Error(`${label} timeout (${resolvedTimeout}ms)`)); }, resolvedTimeout); }) ]); } finally { if (timeoutHandle != null) { globalThis.clearTimeout(timeoutHandle); } } } async function drainResponseWithProgress(response, progressCallback, options = {}) { const total = Number(response.headers.get("content-length")) || 0; const chunkTimeoutMs = parsePositiveInteger(options.chunkTimeoutMs); if (!response.body || typeof response.body.getReader !== "function") { const bytes = new Uint8Array(await response.arrayBuffer()); if (typeof progressCallback === "function") { progressCallback({ loaded: bytes.byteLength, total: total || bytes.byteLength }); } return bytes.byteLength; } const reader = response.body.getReader(); let loaded = 0; let lastBucket = -1; while (true) { const { done, value } = await readStreamChunkWithTimeout( reader, chunkTimeoutMs, "response drain read" ); if (done) { break; } if (!value || value.length === 0) { continue; } loaded += value.length; if (typeof progressCallback === "function") { const effectiveTotal = total || loaded; const bucket = effectiveTotal > 0 ? Math.floor(loaded / effectiveTotal * 100) : -1; if (bucket > lastBucket) { lastBucket = bucket; progressCallback({ loaded, total: effectiveTotal }); } } } if (typeof progressCallback === "function") { progressCallback({ loaded, total: total || loaded }); } return loaded; } function ensureFsDirectory(fs, dirPath) { const slash = dirPath.lastIndexOf("/"); const parent = slash > 0 ? dirPath.slice(0, slash) : "/"; const name = dirPath.slice(slash + 1); const exists = () => { try { const entries = fs.readdir(parent); return Array.isArray(entries) && entries.includes(name); } catch (_) { return false; } }; if (exists()) { return; } try { fs.mkdir(dirPath); } catch (error) { if (!exists()) { throw error; } } } function unlinkFsFile(fs, filePath) { try { fs.unlink(filePath); return true; } catch (_) { return false; } } async function writeResponseToFsFileWithProgress(response, fs, filePath, progressCallback, writeOptions = {}) { const total = parsePositiveInteger(writeOptions.totalBytes) || inferResponseTotalBytes(response, 0); const useBigIntPosition = writeOptions.useBigIntPosition === true; const startOffset = parsePositiveInteger(writeOptions.startOffset); const preservePartialOnError = writeOptions.preservePartialOnError === true; const appendMode = startOffset > 0; const chunkTimeoutMs = parsePositiveInteger(writeOptions.chunkTimeoutMs); const signal = writeOptions.signal || null; const abortMessage = writeOptions.abortMessage || "Model transfer was cancelled."; throwIfAborted(signal, abortMessage); if (!appendMode) { unlinkFsFile(fs, filePath); } if (!response.body || typeof response.body.getReader !== "function") { throwIfAborted(signal, abortMessage); const bytes = new Uint8Array(await response.arrayBuffer()); throwIfAborted(signal, abortMessage); if (appendMode) { const stream2 = fs.open(filePath, "r+"); try { throwIfAborted(signal, abortMessage); const position = useBigIntPosition ? BigInt(startOffset) : startOffset; fs.write(stream2, bytes, 0, bytes.length, position); } finally { fs.close(stream2); } } else { fs.writeFile(filePath, bytes); } const finalLoaded2 = startOffset + bytes.byteLength; if (typeof progressCallback === "function") { progressCallback({ loaded: finalLoaded2, total: total || finalLoaded2 }); } return finalLoaded2; } const reader = response.body.getReader(); const openMode = appendMode ? "r+" : "w"; const stream = fs.open(filePath, openMode); let loaded = 0; let lastBucket = -1; let writePosition = startOffset; let writePositionBigInt = null; try { while (true) { throwIfAborted(signal, abortMessage); const { done, value } = await readStreamChunkWithTimeout( reader, chunkTimeoutMs, "response file read" ); if (done) { throwIfAborted(signal, abortMessage); break; } if (!value || value.length === 0) { continue; } const chunk = value.slice ? value.slice() : new Uint8Array(value); throwIfAborted(signal, abortMessage); if (useBigIntPosition) { if (writePositionBigInt == null) { writePositionBigInt = BigInt(startOffset); } fs.write(stream, chunk, 0, chunk.length, writePositionBigInt); writePositionBigInt += BigInt(chunk.length); } else { fs.write(stream, chunk, 0, chunk.length, writePosition); writePosition += chunk.length; } loaded += chunk.length; if (typeof progressCallback === "function") { const effectiveLoaded = startOffset + loaded; const effectiveTotal = total || effectiveLoaded; const bucket = effectiveTotal > 0 ? Math.floor(effectiveLoaded / effectiveTotal * 100) : -1; if (bucket > lastBucket) { lastBucket = bucket; progressCallback({ loaded: effectiveLoaded, total: effectiveTotal }); } } } } catch (error) { if (signal?.aborted && error?.name !== "AbortError") { error = createAbortError(abortMessage); } try { if (error && typeof error === "object") { error.llamadartLoadedBytes = startOffset + loaded; error.llamadartFilePath = filePath; } } catch (_) { } try { fs.close(stream); } catch (_) { } try { await reader.cancel?.(); } catch (_) { } if (!preservePartialOnError || signal?.aborted || error?.name === "AbortError") { try { fs.unlink(filePath); } catch (_) { } } throw error; } fs.close(stream); const finalLoaded = startOffset + loaded; if (typeof progressCallback === "function") { progressCallback({ loaded: finalLoaded, total: total || finalLoaded }); } return finalLoaded; } // js/src/internal/environment.ts function isSafariUserAgent(userAgent) { if (typeof userAgent !== "string" || userAgent.length === 0) { return false; } const hasSafariToken = /Safari\//.test(userAgent); const hasOtherBrowserToken = /(Chrome|Chromium|CriOS|Edg|OPR|Firefox|FxiOS)\//.test(userAgent); return hasSafariToken && !hasOtherBrowserToken; } function isCrossOriginIsolatedRuntime() { try { if (typeof globalThis.crossOriginIsolated === "boolean") { return globalThis.crossOriginIsolated; } } catch (_) { } return true; } // js/src/internal/image.ts async function decodeImageBytesToRgb(bytes, options = {}) { const sourceBytes = toUint8Array(bytes); if (!sourceBytes || sourceBytes.length === 0) { return null; } if (typeof createImageBitmap !== "function" || typeof Blob !== "function") { return null; } const maxPixelsCandidate = Number(options.maxPixels); const maxPixels = Number.isFinite(maxPixelsCandidate) && maxPixelsCandidate > 0 ? Math.max(65536, Math.min(33554432, Math.trunc(maxPixelsCandidate))) : 0; const maxEdgeCandidate = Number(options.maxEdge); const maxEdge = Number.isFinite(maxEdgeCandidate) && maxEdgeCandidate > 0 ? Math.max(64, Math.min(16384, Math.trunc(maxEdgeCandidate))) : 0; if (maxPixels <= 0 && maxEdge <= 0) { return null; } let bitmap = null; try { const mimeType = typeof options.mimeType === "string" && options.mimeType.length > 0 ? options.mimeType : "image/png"; const blob = new Blob([sourceBytes], { type: mimeType }); bitmap = await createImageBitmap(blob); const sourceWidth = Math.max(1, Math.trunc(Number(bitmap.width) || 0)); const sourceHeight = Math.max(1, Math.trunc(Number(bitmap.height) || 0)); let scale = 1; if (maxPixels > 0) { const sourcePixels = sourceWidth * sourceHeight; if (sourcePixels > maxPixels) { scale = Math.min(scale, Math.sqrt(maxPixels / sourcePixels)); } } if (maxEdge > 0) { const sourceLongest = Math.max(sourceWidth, sourceHeight); if (sourceLongest > maxEdge) { scale = Math.min(scale, maxEdge / sourceLongest); } } const width = Math.max(1, Math.round(sourceWidth * scale)); const height = Math.max(1, Math.round(sourceHeight * scale)); let canvas = null; let context = null; if (typeof OffscreenCanvas === "function") { canvas = new OffscreenCanvas(width, height); context = canvas.getContext("2d", { alpha: false, willReadFrequently: true }); } if (!context && typeof document !== "undefined" && typeof document.createElement === "function") { canvas = document.createElement("canvas"); canvas.width = width; canvas.height = height; context = canvas.getContext("2d", { alpha: false, willReadFrequently: true }); } if (!context) { return null; } context.drawImage(bitmap, 0, 0, width, height); let encodedBytes = null; if (canvas && "convertToBlob" in canvas && typeof canvas.convertToBlob === "function") { const encodedBlob = await canvas.convertToBlob({ type: "image/png" }); if (encodedBlob) { encodedBytes = new Uint8Array(await encodedBlob.arrayBuffer()); } } if (!encodedBytes && canvas && "toBlob" in canvas && typeof canvas.toBlob === "function" && typeof Promise === "function") { const encodedBlob = await new Promise((resolve) => { canvas.toBlob((value) => { resolve(value || null); }, "image/png"); }); if (encodedBlob) { encodedBytes = new Uint8Array(await encodedBlob.arrayBuffer()); } } if (!encodedBytes || encodedBytes.length === 0) { return null; } return { bytes: encodedBytes, width, height, sourceWidth, sourceHeight, resized: width !== sourceWidth || height !== sourceHeight }; } catch (_) { return null; } finally { try { bitmap?.close?.(); } catch (_) { } } } // js/src/internal/next_token_scores.ts function nextTokenCandidateIds(value) { if (value == null) { return []; } const isList = Array.isArray(value) || ArrayBuffer.isView(value) && !(value instanceof DataView); if (!isList) { throw new TypeError("Next-token candidates must be an array or typed array of token ids."); } return Array.from(value, (item, index) => { if (!isInt32(item)) { throw new TypeError(`Next-token candidates[${index}] is ${String(item)}; expected a 32-bit integer.`); } return item; }); } function scoredTokensFrom(entries) { if (!Array.isArray(entries)) { return []; } return entries.map((entry) => ({ token: Number(entry?.token), bytes: Uint8Array.from(Array.isArray(entry?.bytes) ? entry.bytes : []), logprob: typeof entry?.logprob === "number" ? entry.logprob : -Infinity })); } // js/src/internal/text.ts function buildPromptFromMessages(messages, addAssistant) { const lines = []; for (const msg of messages || []) { const role = String(msg?.role ?? "user"); const content = String(msg?.content ?? ""); lines.push(`${role}: ${content}`); } if (addAssistant) { lines.push("assistant: "); } return lines.join("\n"); } function looksLikeCorruptedGeneration(text) { if (typeof text !== "string" || text.length === 0) { return false; } const normalized = text.trim(); if (normalized.length === 0) { return false; } const unusedTokens = text.match(//g) || []; if (unusedTokens.length >= 4) { return true; } const tokenLikeTags = text.match(/<[^>]{1,40}>/g) || []; if (tokenLikeTags.length >= 8) { return true; } const compact = text.replace(/\s+/g, ""); if (compact.length === 0) { return false; } const tagRun = compact.match(/(?:<[^>]{2,32}>){6,}/); if (tagRun) { return true; } const alphaNum = (normalized.match(/[A-Za-z0-9]/g) || []).length; const printable = (normalized.match(/[\x20-\x7E]/g) || []).length; const angleBrackets = (normalized.match(/[<>]/g) || []).length; const alphaNumRatio = alphaNum / normalized.length; const printableRatio = printable / normalized.length; const bracketRatio = angleBrackets / normalized.length; if (normalized.length >= 24 && printableRatio > 0.95 && alphaNumRatio < 0.18) { return true; } if (normalized.length >= 24 && bracketRatio > 0.25) { return true; } return false; } function trimUnstableUtf8Tail(text) { if (typeof text !== "string" || text.length === 0) { return ""; } let end = text.length; while (end > 0 && text.charCodeAt(end - 1) === 65533) { end -= 1; } if (end > 0) { const tail = text.charCodeAt(end - 1); if (tail >= 55296 && tail <= 56319) { end -= 1; } } return end === text.length ? text : text.slice(0, end); } // js/src/runtime.ts var textEncoder = new TextEncoder(); var LlamaWebGpuBridgeRuntime = class { constructor(config = {}) { this._config = config; this._core = null; this._backendLabels = []; this._gpuActive = false; this._modelPath = null; this._modelPaths = []; this._modelBytes = 0; this._mmProjPath = null; this._mmSupportsVision = false; this._mmSupportsAudio = false; this._mediaFileCounter = 0; this._stateFileCounter = 0; this._decisionFileCounter = 0; this._loraFileCounter = 0; this._loraAdaptersLoaded = false; this._stagedMediaPaths = []; this._nCtx = 4096; this._abortRequested = false; this._textToSpeechActive = false; this._textToSpeechDone = null; this._resolveTextToSpeechDone = null; this._runtimeNotes = []; this._threadPoolSizeHint = Number(config.threadPoolSize) > 0 ? Math.max(1, Math.trunc(Number(config.threadPoolSize))) : null; const requestedThreads = Number(config.threads) > 0 ? Number(config.threads) : this._resolveAutoThreadCount(); this._threads = this._capThreadsToPool(requestedThreads); const requestedThreadsBatch = Number(config.threadsBatch) > 0 ? Number(config.threadsBatch) : this._threads; this._threadsBatch = this._capThreadsToPool( requestedThreadsBatch, { noteTag: "threads_batch_capped_pool" } ); this._nBatch = Number(config.nBatch) > 0 ? Math.max(32, Math.trunc(Number(config.nBatch))) : 0; this._nUbatch = Number(config.nUbatch) > 0 ? Math.max(32, Math.trunc(Number(config.nUbatch))) : 0; this._nGpuLayers = Number.isFinite(config.nGpuLayers) ? Number(config.nGpuLayers) : -1; this._nSeqMax = 0; this._useMmap = false; this._useMlock = false; this._flashAttention = -1; this._cacheTypeK = 1; this._cacheTypeV = 1; this._kvUnified = -1; this._ropeFrequencyBase = 0; this._ropeFrequencyScale = 0; this._splitMode = -1; this._mainGpu = -1; this._loadMtp = false; this._speculativeRollbackTokenMax = 0; this._isSafari = isSafariUserAgent(this._config.userAgent ?? globalThis.navigator?.userAgent ?? ""); this._coreVariant = "uninitialized"; this._preferMemory64 = this._config.preferMemory64 !== false; this._modelSource = "network"; this._modelCacheState = "disabled"; this._modelCacheName = defaultModelCacheName; this._loadedModelUrl = null; this._mmProjSourceUrl = null; this._draftModel = null; this._speculativeFileCounter = 0; this._suppressedWarmupWarningCount = 0; this._didReportWarmupWarningSuppression = false; this._remoteFetchThresholdBytes = Number(config.remoteFetchThresholdBytes) > 0 ? Number(config.remoteFetchThresholdBytes) : 1900 * 1024 * 1024; this._remoteFetchChunkBytes = Number(config.remoteFetchChunkBytes) > 0 ? Number(config.remoteFetchChunkBytes) : 16 * 1024 * 1024; this._mediaMaxImagePixels = Number(config.mediaMaxImagePixels) > 0 ? Math.max(65536, Math.min(33554432, Math.trunc(Number(config.mediaMaxImagePixels)))) : 1024 * 1024; this._mediaMaxImageEdge = Number(config.mediaMaxImageEdge) > 0 ? Math.max(64, Math.min(16384, Math.trunc(Number(config.mediaMaxImageEdge)))) : 1280; this._disableImageDownscale = config.disableImageDownscale === true; this._activeTransferAbortController = null; this._lastCoreErrorText = ""; this._lastCoreErrorHint = ""; this._logLevel = Number.isFinite(config.logLevel) ? Math.max(0, Math.min(4, Math.trunc(config.logLevel))) : 2; } static supportsSafariAdaptiveGpu = true; _pushRuntimeNote(note) { if (typeof note !== "string" || note.length === 0) { return; } if (!Array.isArray(this._runtimeNotes)) { this._runtimeNotes = []; } if (!this._runtimeNotes.includes(note)) { this._runtimeNotes.push(note); } } _detectThreadPoolSizeFromCore() { if (!this._coreSupportsPthreads()) { return 1; } const core = this._core; if (!core || typeof core !== "object") { return null; } try { if (typeof core.ccall === "function") { const compiledPoolSize = Number( core.ccall("llamadart_webgpu_pthread_pool_size", "number", [], []) ); if (Number.isFinite(compiledPoolSize) && compiledPoolSize > 0) { return Math.max(1, Math.trunc(compiledPoolSize)); } } } catch (_) { } try { const pThread = core.PThread; if (!pThread || typeof pThread !== "object") { return null; } const unused = Array.isArray(pThread.unusedWorkers) ? pThread.unusedWorkers.length : 0; const running = Array.isArray(pThread.runningWorkers) ? pThread.runningWorkers.length : 0; const total = unused + running; return total > 0 ? total : null; } catch (_) { return null; } } _coreSupportsPthreads() { const core = this._core; if (!core || typeof core !== "object") { return false; } try { if (typeof core.ccall === "function") { const compiledWithPthreads = Number( core.ccall("llamadart_webgpu_supports_pthreads", "number", [], []) ); if (Number.isFinite(compiledWithPthreads)) { return compiledWithPthreads !== 0; } } } catch (_) { } try { const wasmBuffer = core.wasmMemory?.buffer; if (typeof SharedArrayBuffer === "function" && wasmBuffer instanceof SharedArrayBuffer) { return true; } } catch (_) { } try { const heapBuffer = core.HEAP8?.buffer || core.HEAPU8?.buffer; if (typeof SharedArrayBuffer === "function" && heapBuffer instanceof SharedArrayBuffer) { return true; } } catch (_) { } try { const pThread = core.PThread; if (!pThread || typeof pThread !== "object") { return false; } return Array.isArray(pThread.unusedWorkers) || Array.isArray(pThread.runningWorkers) || typeof pThread.allocateUnusedWorker === "function"; } catch (_) { return false; } } _syncThreadPoolSizeHintFromCore() { const detected = this._detectThreadPoolSizeFromCore(); if (!Number.isFinite(detected) || detected <= 0) { return; } this._threadPoolSizeHint = Math.max(1, Math.trunc(detected)); } _resolveAutoThreadCount() { const hardwareThreads = Number(globalThis.navigator?.hardwareConcurrency); if (Number.isFinite(hardwareThreads) && hardwareThreads > 0) { return Math.max(1, Math.min(8, Math.trunc(hardwareThreads))); } return 4; } _capThreadsToPool(candidate, { noteTag = "threads_capped_pool" } = {}) { let resolved = Number(candidate); if (!Number.isFinite(resolved) || resolved <= 0) { resolved = 1; } resolved = Math.max(1, Math.trunc(resolved)); const poolSize = Number(this._threadPoolSizeHint); if (Number.isFinite(poolSize) && poolSize > 0 && resolved > poolSize) { if (noteTag) { this._pushRuntimeNote(`${noteTag}:${poolSize}`); } return poolSize; } return resolved; } _isVerboseWarmupWarning(text) { const lowered = String(text || "").toLowerCase(); if (lowered.length === 0) { return false; } if (lowered.includes("warmup:")) { return true; } if (lowered.includes("please report this on github as an issue")) { return true; } if (lowered.includes("warning: ref:")) { return true; } if (lowered.includes("github.com/ggml-org/llama.cpp/pull/")) { return true; } if (lowered.includes("****************")) { return true; } return false; } _emitSuppressedWarmupWarningSummaryIfNeeded() { if (this._suppressedWarmupWarningCount <= 0) { return; } if (this._logLevel <= 2) { this._emitLogger( "log", `info: suppressed ${this._suppressedWarmupWarningCount} verbose warmup log lines (set bridge/runtime log level to Debug to inspect full warmup trace).` ); } this._suppressedWarmupWarningCount = 0; } _shouldAttemptGenerationRecovery(errorText, options = {}, generated = 0) { if (generated > 0) { return false; } if (this._nGpuLayers <= 0) { return false; } if (this._coreVariant !== "wasm64") { return false; } if (!hasModelSource(this._loadedModelUrl)) { return false; } if (options._llamadartGenerationRecoveryAttempted === true) { return false; } if (Array.isArray(options.parts) && options.parts.length > 0) { return false; } const lowered = String(errorText || "").toLowerCase(); if (lowered.includes("failed to decode")) { return true; } if (lowered.includes("failed to compute graph")) { return true; } if (lowered.includes("ggml_backend_sched_graph_compute_async failed")) { return true; } return false; } _isContextLimitGenerationError(errorText = "") { const lowered = [ String(errorText || ""), String(this._lastCoreErrorText || ""), String(this._lastCoreErrorHint || "") ].join(" ").toLowerCase(); return lowered.includes("failed to find a memory slot for batch") || lowered.includes("failed to prepare attention batches") || lowered.includes("context overflow") || lowered.includes("context full") || lowered.includes("kv cache full") || lowered.includes("insufficient kv") || lowered.includes("no kv slot"); } async _recoverGenerationWithCpuFallback(options = {}) { const modelUrl = cloneModelSource(this._loadedModelUrl); if (!hasModelSource(modelUrl)) { return false; } if (this._loraAdaptersLoaded) { this._runtimeNotes.push("generation_recovery_cpu_skipped_lora"); return false; } this._runtimeNotes.push("generation_recovery_cpu_attempt"); this._emitLogger( "warn", "warning: generation failed on wasm64/WebGPU; retrying by reloading model with CPU fallback for stability." ); const previousPreferMemory64 = this._preferMemory64; const previousMMProjSourceUrl = this._mmProjSourceUrl; const previousDraftModel = this._draftModel; const resolvedLoadState = this._resolvedLoadState(); try { this._preferMemory64 = false; await this.loadModelFromUrl(modelUrl, { nCtx: this._nCtx, nThreads: this._threads, nGpuLayers: 0, useCache: true, forceRemoteFetchBackend: false, remoteFetchChunkBytes: this._resolveRemoteFetchChunkBytes(options), safariGpuProbe: false, signal: options.signal }, resolvedLoadState); if (typeof previousMMProjSourceUrl === "string" && previousMMProjSourceUrl.length > 0) { try { await this.loadMultimodalProjector(previousMMProjSourceUrl); } catch (_) { this._runtimeNotes.push("generation_recovery_mmproj_reload_failed"); } } if (previousDraftModel) { try { await this.loadDraftModel(previousDraftModel.url, previousDraftModel.options); } catch (_) { this._runtimeNotes.push("generation_recovery_draft_reload_failed"); } } this._runtimeNotes.push("generation_recovery_cpu_applied"); return true; } catch (_) { this._runtimeNotes.push("generation_recovery_cpu_failed"); return false; } finally { this._preferMemory64 = previousPreferMemory64; } } _loggerFor(level) { const logger = this._config?.logger; const fallback = typeof console !== "undefined" ? console : null; if (logger && typeof logger[level] === "function") { return logger[level].bind(logger); } if (!fallback) { return () => { }; } if (typeof fallback[level] === "function") { return fallback[level].bind(fallback); } if (typeof fallback.log === "function") { return fallback.log.bind(fallback); } return () => { }; } _shouldEmitLoggerLevel(level) { const current = Number(this._logLevel); if (!Number.isFinite(current) || current < 0) { return true; } const threshold = logThresholdForConfiguredLevel( Math.max(0, Math.min(4, Math.trunc(current))) ); if (threshold > 3) { return false; } return logLevelForName(level) >= threshold; } _emitLogger(level, message) { if (!this._shouldEmitLoggerLevel(level)) { return; } try { this._loggerFor(level)(message); } catch (_) { } } _classifyCoreErrorLine(text) { const trimmed = String(text ?? "").trim(); if (trimmed.length === 0) { return "ignore"; } if (this._isVerboseWarmupWarning(trimmed)) { return "warmup"; } const lowered = trimmed.toLowerCase(); if (lowered.startsWith("warning") || lowered.startsWith("warn:") || lowered.includes(" warning:")) { return "warn"; } if (lowered.startsWith("error") || lowered.startsWith("err:") || lowered.includes(" error:") || lowered.includes("failed") || lowered.includes("exception") || lowered.includes("abort") || lowered.includes("fatal") || lowered.includes("out of memory") || lowered.includes("invalid")) { return "error"; } return "info"; } _applyCoreLogLevel() { if (!this._core) { return; } try { this._core.ccall( "llamadart_webgpu_set_log_level", null, ["number"], [this._logLevel] ); } catch (_) { } } _coreErrorMessage(prefix, fallbackCode = 0) { try { const err = this._core?.ccall("llamadart_webgpu_last_error", "string", [], []); if (err) { return `${prefix}: ${err}`; } } catch (_) { } return `${prefix} (code=${fallbackCode})`; } _resolveCacheName(options = {}) { if (typeof options.cacheName === "string" && options.cacheName.trim().length > 0) { return options.cacheName.trim(); } if (typeof this._config.cacheName === "string" && this._config.cacheName.trim().length > 0) { return this._config.cacheName.trim(); } return defaultModelCacheName; } _isWorkerRuntime() { return typeof WorkerGlobalScope !== "undefined" && globalThis instanceof WorkerGlobalScope; } _resolveRemoteFetchThresholdBytes(options = {}) { const candidate = Number(options.remoteFetchThresholdBytes); if (Number.isFinite(candidate) && candidate > 0) { return Math.trunc(candidate); } return Math.trunc(this._remoteFetchThresholdBytes); } _resolveRemoteFetchChunkBytes(options = {}) { const candidate = Number(options.remoteFetchChunkBytes); if (Number.isFinite(candidate) && candidate > 0) { return Math.max(16 * 1024, Math.trunc(candidate)); } return Math.max(16 * 1024, Math.trunc(this._remoteFetchChunkBytes)); } _canUseRemoteFetchBackend(options = {}) { if (options.forceRemoteFetchBackend === false) { return false; } if (!this._isWorkerRuntime()) { return false; } if (options.forceRemoteFetchBackend === true) { return true; } return this._config.allowAutoRemoteFetchBackend === true; } async _tryHeadContentLength(url) { function parseSizeFromHeaders(headers) { if (!headers) { return 0; } const length = Number(headers.get("content-length")) || 0; if (length > 0) { return length; } const linkedSize = Number(headers.get("x-linked-size")) || 0; if (linkedSize > 0) { return linkedSize; } const contentRange = String(headers.get("content-range") || ""); const slash = contentRange.lastIndexOf("/"); if (slash >= 0 && slash + 1 < contentRange.length) { const total = Number(contentRange.slice(slash + 1)) || 0; if (total > 0) { return total; } } return 0; } try { const response = await this._fetchWithTimeout(url, { method: "HEAD", cache: "no-store" }, 45e3); if (!response.ok) { throw new Error("HEAD request failed"); } const size = parseSizeFromHeaders(response.headers); if (size > 0) { return size; } } catch (_) { } try { const probe = await this._fetchWithTimeout(url, { headers: { Range: "bytes=0-0" }, cache: "no-store" }, 45e3); if (!(probe.ok || probe.status === 206)) { return null; } const size = parseSizeFromHeaders(probe.headers); if (size > 0) { this._runtimeNotes.push("model_fetch_size_probe"); return size; } } catch (_) { } return null; } async _resolveRemoteFetchUrl(url) { function parseSizeFromHeaders(headers) { if (!headers) { return 0; } const contentRange = String(headers.get("content-range") || ""); const slash = contentRange.lastIndexOf("/"); if (slash >= 0 && slash + 1 < contentRange.length) { const total = Number(contentRange.slice(slash + 1)) || 0; if (total > 0) { return total; } } const linkedSize = Number(headers.get("x-linked-size")) || 0; if (linkedSize > 0) { return linkedSize; } const length = Number(headers.get("content-length")) || 0; if (length > 0) { return length; } return 0; } try { const probe = await this._fetchWithTimeout(url, { headers: { Range: "bytes=0-0" }, cache: "no-store" }, 45e3); if (!(probe.ok || probe.status === 206)) { return null; } const resolvedUrl = typeof probe.url === "string" && probe.url.length > 0 ? probe.url : null; const sizeBytes = parseSizeFromHeaders(probe.headers); return { resolvedUrl, sizeBytes: sizeBytes > 0 ? sizeBytes : null }; } catch (_) { return null; } } _resolveNativeLoadOptions(options = {}) { this._nSeqMax = parsePositiveInteger(options.nSeqMax); this._useMmap = parseBooleanFlag(options.useMmap, false); this._useMlock = parseBooleanFlag(options.useMlock, false); this._flashAttention = parseEnumValue(options.flashAttention, [-1, 0, 1], -1); this._cacheTypeK = parseEnumValue(options.cacheTypeK, [1, 2, 8], 1); this._cacheTypeV = parseEnumValue(options.cacheTypeV, [1, 2, 8], 1); this._kvUnified = parseOptionalBooleanFlag(options.kvUnified); this._ropeFrequencyBase = parsePositiveNumber(options.ropeFrequencyBase); this._ropeFrequencyScale = parsePositiveNumber(options.ropeFrequencyScale); this._splitMode = parseEnumValue(options.splitMode, [0, 1, 2, 3], -1); this._mainGpu = parseInteger(options.mainGpu, -1); if (this._mainGpu < 0) { this._mainGpu = -1; } this._loadMtp = parseBooleanFlag(options.loadMtp, false); this._speculativeRollbackTokenMax = parsePositiveInteger(options.speculativeRollbackTokenMax); const wantsQuantizedKvCache = this._cacheTypeK !== 1 || this._cacheTypeV !== 1; if (this._flashAttention === 0 && wantsQuantizedKvCache) { throw new Error( "Non-F16 KV cache requires flashAttention to be auto or enabled." ); } if (this._flashAttention === -1 && wantsQuantizedKvCache) { this._flashAttention = 1; this._runtimeNotes.push("flash_attention:auto_enabled_for_kv_cache"); } if (this._kvUnified < 0 && this._nSeqMax > 1) { this._kvUnified = 1; this._runtimeNotes.push("kv_unified:auto_enabled_for_sequences"); } } _nativeLoadOptionValues() { return [ this._nSeqMax, this._useMmap ? 1 : 0, this._useMlock ? 1 : 0, this._flashAttention, this._cacheTypeK, this._cacheTypeV, this._kvUnified, this._ropeFrequencyBase, this._ropeFrequencyScale, this._splitMode, this._mainGpu, this._loadMtp ? 1 : 0, this._speculativeRollbackTokenMax ]; } _nativeLoadOptionTypes() { return this._nativeLoadOptionValues().map(() => "number"); } _resolvedLoadState() { return { _nCtx: this._nCtx, _threads: this._threads, _threadsBatch: this._threadsBatch, _nBatch: this._nBatch, _nUbatch: this._nUbatch, _nSeqMax: this._nSeqMax, _useMmap: this._useMmap, _useMlock: this._useMlock, _flashAttention: this._flashAttention, _cacheTypeK: this._cacheTypeK, _cacheTypeV: this._cacheTypeV, _kvUnified: this._kvUnified, _ropeFrequencyBase: this._ropeFrequencyBase, _ropeFrequencyScale: this._ropeFrequencyScale, _splitMode: this._splitMode, _mainGpu: this._mainGpu, _loadMtp: this._loadMtp, _speculativeRollbackTokenMax: this._speculativeRollbackTokenMax }; } async _tryLoadModelFromRemoteFetchBackend(core, url, options = {}) { if (!this._canUseRemoteFetchBackend(options)) { return { loaded: false, sizeBytes: null }; } const thresholdBytes = this._resolveRemoteFetchThresholdBytes(options); const chunkBytes = this._resolveRemoteFetchChunkBytes(options); const forceRemote = options.forceRemoteFetchBackend === true; let sizeBytes = Number(options.modelBytesHint); if (!Number.isFinite(sizeBytes) || sizeBytes <= 0) { sizeBytes = await this._tryHeadContentLength(url); } if (!forceRemote) { if (Number.isFinite(sizeBytes) && sizeBytes > 0 && sizeBytes < thresholdBytes) { this._runtimeNotes.push("model_fetch_backend_skipped_small"); return { loaded: false, sizeBytes }; } if (!Number.isFinite(sizeBytes) || sizeBytes <= 0) { this._runtimeNotes.push("model_fetch_backend_size_unknown"); this._runtimeNotes.push("model_fetch_backend_unknown_size_attempt"); } } this._runtimeNotes.push("model_fetch_backend_attempt"); this._runtimeNotes.push(`model_fetch_chunk:${chunkBytes}`); this._lastCoreErrorHint = ""; let remoteFetchUrl = url; const resolvedProbe = await this._resolveRemoteFetchUrl(url); if (resolvedProbe?.resolvedUrl) { remoteFetchUrl = resolvedProbe.resolvedUrl; if (remoteFetchUrl !== url) { this._runtimeNotes.push("model_fetch_backend_resolved_url"); } } if ((!Number.isFinite(sizeBytes) || sizeBytes <= 0) && Number.isFinite(resolvedProbe?.sizeBytes) && resolvedProbe.sizeBytes > 0) { sizeBytes = resolvedProbe.sizeBytes; } try { globalThis.__llamadartFetchBackendLastError = null; } catch (_) { } if (typeof options.progressCallback === "function") { options.progressCallback({ loaded: 0, total: Number.isFinite(sizeBytes) ? sizeBytes : 0 }); } try { const rc = Number( await core.ccall( "llamadart_webgpu_load_model_from_url", "number", [ "string", "number", "number", "number", "number", "number", "number", "number", ...this._nativeLoadOptionTypes() ], [ remoteFetchUrl, this._nCtx, this._threads, this._threadsBatch, this._nBatch, this._nUbatch, this._nGpuLayers, chunkBytes, ...this._nativeLoadOptionValues() ], { async: true } ) ); if (rc !== 0) { throw new Error(this._coreErrorMessage("Fetch-backed model load failed", rc)); } this._modelSource = "network-fetch"; this._modelPath = null; this._modelBytes = Number.isFinite(sizeBytes) && sizeBytes > 0 ? Math.trunc(sizeBytes) : 1; this._runtimeNotes.push("model_source_fetch_backend"); if (typeof options.progressCallback === "function") { const resolved = Number.isFinite(sizeBytes) && sizeBytes > 0 ? sizeBytes : 1; options.progressCallback({ loaded: resolved, total: resolved }); } return { loaded: true, sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null }; } catch (error) { const text = String(error || "").toLowerCase(); if (text.includes("aborted(native code called abort())")) { try { const stderrText = String( this._lastCoreErrorHint || this._lastCoreErrorText || "" ).trim(); if (stderrText.length > 0) { const token = stderrText.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, ""); if (token.length > 0) { this._runtimeNotes.push(`model_core_stderr:${token}`); } } } catch (_) { } try { const fetchError = String( globalThis.__llamadartFetchBackendLastError || "" ).trim(); if (fetchError.length > 0) { const token = fetchError.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, ""); if (token.length > 0) { this._runtimeNotes.push(`model_fetch_js_error:${token}`); } } } catch (_) { } try { const stats = globalThis.__llamadartFetchBackendStats; if (stats && typeof stats === "object") { const reads = Number(stats.reads) || 0; const getSize = Number(stats.getSize) || 0; const ranges = Number(stats.ranges) || 0; const fallbacks = Number(stats.wholeFileFallbacks) || 0; const errors = Number(stats.errors) || 0; this._runtimeNotes.push( `model_fetch_stats:r${reads}_s${getSize}_q${ranges}_f${fallbacks}_e${errors}` ); } } catch (_) { } try { const coreError = String( this._core?.ccall("llamadart_webgpu_last_error", "string", [], []) || "" ).trim(); if (coreError.length > 0) { const token = coreError.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, ""); if (token.length > 0) { this._runtimeNotes.push(`model_fetch_core_error:${token}`); } } } catch (_) { } this._runtimeNotes.push("model_fetch_backend_abort"); throw error; } if (text.includes("fetch-backed model load failed")) { this._runtimeNotes.push("model_fetch_backend_failed"); return { loaded: false, sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null }; } if (text.includes("load_model_from_url") && (text.includes("not found") || text.includes("undefined symbol") || text.includes("missing function"))) { this._runtimeNotes.push("model_fetch_backend_unavailable"); return { loaded: false, sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null }; } if (text.includes("worker-thread bridge runtime")) { this._runtimeNotes.push("model_fetch_backend_requires_worker"); return { loaded: false, sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null }; } throw error; } } _beginTransferAbortController() { if (typeof AbortController !== "function") { this._activeTransferAbortController = null; return null; } if (this._activeTransferAbortController) { try { this._activeTransferAbortController.abort(); } catch (_) { } } const controller = new AbortController(); this._activeTransferAbortController = controller; return controller; } _clearTransferAbortController(controller) { if (this._activeTransferAbortController === controller) { this._activeTransferAbortController = null; } } _resolveFetchTimeoutMs(options = {}, defaultTimeoutMs = 18e4) { const configured = Number(options.fetchTimeoutMs); if (Number.isFinite(configured) && configured > 0) { return Math.max(1e4, Math.min(18e5, Math.trunc(configured))); } return defaultTimeoutMs; } _resolveStreamChunkTimeoutMs(options = {}, defaultTimeoutMs = 9e4) { const configured = Number(options.streamChunkTimeoutMs); if (Number.isFinite(configured) && configured > 0) { return Math.max(5e3, Math.min(3e5, Math.trunc(configured))); } return defaultTimeoutMs; } _resolveCoreInitTimeoutMs() { const configured = Number(this._config.coreInitTimeoutMs); if (!Number.isFinite(configured) || configured <= 0) { return 9e4; } return Math.max(1e4, Math.min(6e5, Math.trunc(configured))); } _resolveMediaImageMaxPixels(options = {}) { if (options.disableImageDownscale === true || this._disableImageDownscale) { return 0; } const configured = Number(options.mediaMaxImagePixels); if (Number.isFinite(configured) && configured > 0) { return Math.max(65536, Math.min(33554432, Math.trunc(configured))); } return this._mediaMaxImagePixels; } _resolveMediaImageMaxEdge(options = {}) { if (options.disableImageDownscale === true || this._disableImageDownscale) { return 0; } const configured = Number(options.mediaMaxImageEdge); if (Number.isFinite(configured) && configured > 0) { return Math.max(64, Math.min(16384, Math.trunc(configured))); } return this._mediaMaxImageEdge; } async _fetchWithTimeout(url, init = {}, timeoutMs = 0) { const resolvedTimeout = Number(timeoutMs); if (!Number.isFinite(resolvedTimeout) || resolvedTimeout <= 0) { return fetch(url, init); } if (typeof AbortController !== "function") { return Promise.race([ fetch(url, init), new Promise((_, reject) => { globalThis.setTimeout( () => reject(new Error(`fetch timeout (${resolvedTimeout}ms)`)), resolvedTimeout ); }) ]); } const timeoutController = new AbortController(); const externalSignal = init.signal; let didTimeout = false; let timeoutHandle = null; const onExternalAbort = () => { try { timeoutController.abort(); } catch (_) { } }; if (externalSignal && typeof externalSignal.addEventListener === "function") { externalSignal.addEventListener("abort", onExternalAbort, { once: true }); if (externalSignal.aborted) { onExternalAbort(); } } timeoutHandle = globalThis.setTimeout(() => { didTimeout = true; try { timeoutController.abort(); } catch (_) { } }, resolvedTimeout); try { return await fetch(url, { ...init, signal: timeoutController.signal }); } catch (error) { if (didTimeout) { throw new Error(`fetch timeout (${resolvedTimeout}ms)`); } throw error; } finally { if (timeoutHandle != null) { globalThis.clearTimeout(timeoutHandle); } if (externalSignal && typeof externalSignal.removeEventListener === "function") { externalSignal.removeEventListener("abort", onExternalAbort); } } } _unlinkDecisionFile(path) { if (!this._core || typeof path !== "string" || path.length === 0) { return; } try { this._core.FS.unlink(path); } catch (_) { } } _deleteFsFile(path) { if (!this._core || typeof path !== "string" || path.length === 0) { return false; } return unlinkFsFile(this._core.FS, path); } _releaseModelFiles() { const paths = Array.isArray(this._modelPaths) && this._modelPaths.length > 0 ? [...this._modelPaths] : this._modelPath ? [this._modelPath] : []; let removed = 0; for (const path of paths) { if (this._deleteFsFile(path)) { removed += 1; } } this._modelPaths = []; return removed; } // A load replaces the current model. Free it, its projector and their FS files // before downloading the next model so both never occupy the wasm heap at once. // The core refuses while a generation or speech synthesis is still active, // which leaves the current model untouched. _releaseLoadedModel(core) { const projectorPath = this._mmProjPath; const hasModelFiles = Array.isArray(this._modelPaths) && this._modelPaths.length > 0; if (this._modelBytes <= 0 && !projectorPath && !hasModelFiles) { return false; } const rc = Number(core.ccall("llamadart_webgpu_free_model", "number", [], [])); if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to release the loaded model", rc)); } this._loraAdaptersLoaded = false; this._clearStagedMediaFiles(); this._deleteFsFile(projectorPath); this._releaseModelFiles(); this._modelPath = null; this._modelBytes = 0; this._modelSource = "network"; this._mmProjPath = null; this._mmSupportsVision = false; this._mmSupportsAudio = false; this._draftModel = null; this._runtimeNotes.push("previous_model_released"); return true; } async _getCachedModelResponse(url, options = {}) { let useCache = options.useCache !== false; const forceRefresh = options.force === true; const requireReadableStream = options.requireReadableStream === true; const requestHeaders = options.requestHeaders && typeof options.requestHeaders === "object" ? options.requestHeaders : null; if (requestHeaders && Object.keys(requestHeaders).length > 0) { useCache = false; } const fetchOptions = { ...options.signal ? { signal: options.signal } : {}, ...requestHeaders ? { headers: requestHeaders } : {} }; const fetchTimeoutMs = this._resolveFetchTimeoutMs(options, 18e4); this._modelSource = "network"; this._modelCacheState = useCache ? "unavailable" : "disabled"; this._modelCacheName = this._resolveCacheName(options); if (!useCache) { const response = await this._fetchWithTimeout(url, { cache: "no-store", ...fetchOptions }, fetchTimeoutMs); this._modelCacheState = "disabled"; if (requireReadableStream) { this._runtimeNotes.push( hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream" ); } return response; } if (!globalThis.caches || typeof globalThis.caches.open !== "function") { this._modelCacheState = "unavailable"; const response = await this._fetchWithTimeout(url, fetchOptions, fetchTimeoutMs); if (requireReadableStream) { this._runtimeNotes.push( hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream" ); } return response; } const cacheKey = normalizeAbsoluteUrl(url); try { const cache = await globalThis.caches.open(this._modelCacheName); const cached = forceRefresh ? null : await cache.match(cacheKey); if (cached) { const cacheHasStream = hasReadableResponseStream(cached); if (!requireReadableStream || cacheHasStream) { this._modelSource = "cache"; this._modelCacheState = "hit"; this._runtimeNotes.push("model_cache_hit"); if (requireReadableStream) { this._runtimeNotes.push("model_cache_stream"); } return cached; } this._runtimeNotes.push("model_cache_hit_no_stream"); this._modelSource = "network"; this._modelCacheState = "refresh"; const refreshed = await this._fetchWithTimeout(url, { cache: "no-store", ...fetchOptions }, fetchTimeoutMs); if (refreshed.ok) { try { await cache.put(cacheKey, refreshed.clone()); this._modelCacheState = "stored"; this._runtimeNotes.push("model_cache_stored"); } catch (_) { this._modelCacheState = "store_failed"; this._runtimeNotes.push("model_cache_store_failed"); } } this._runtimeNotes.push( hasReadableResponseStream(refreshed) ? "model_network_stream" : "model_network_no_stream" ); return refreshed; } this._modelCacheState = forceRefresh ? "refresh" : "miss"; const response = await this._fetchWithTimeout(url, fetchOptions, fetchTimeoutMs); if (response.ok) { try { await cache.put(cacheKey, response.clone()); this._modelCacheState = "stored"; this._runtimeNotes.push("model_cache_stored"); } catch (_) { this._modelCacheState = "store_failed"; this._runtimeNotes.push("model_cache_store_failed"); } } if (requireReadableStream) { this._runtimeNotes.push( hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream" ); } return response; } catch (_) { this._modelCacheState = "error"; this._runtimeNotes.push("model_cache_error"); const response = await this._fetchWithTimeout(url, fetchOptions, fetchTimeoutMs); if (requireReadableStream) { this._runtimeNotes.push( hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream" ); } return response; } } async prefetchModelToCache(url, options = {}) { const useCache = options.useCache !== false; this._modelSource = "network"; this._modelCacheState = useCache ? "unavailable" : "disabled"; this._modelCacheName = this._resolveCacheName(options); const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null; const modelUrls = expandModelShardUrls(url); if (modelUrls.length === 0) { throw new Error("Model URL is empty."); } if (modelUrls.length > 1) { this._runtimeNotes.push(`model_split_cache_prefetch:${modelUrls.length}`); } const shardLoaded = new Array(modelUrls.length).fill(0); const shardTotals = new Array(modelUrls.length).fill(0); const emitAggregateProgress = () => { if (!progressCallback) { return; } const loaded = sumProgressValues(shardLoaded); const total = sumProgressValues(shardTotals); progressCallback({ loaded, total: total > 0 ? total : loaded }); }; const controller = this._beginTransferAbortController(); try { const fetchOptions = { cache: "no-store", ...controller?.signal ? { signal: controller.signal } : {} }; const fetchTimeoutMs = this._resolveFetchTimeoutMs(options, 18e4); const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs(options, 9e4); const cache = useCache && globalThis.caches && typeof globalThis.caches.open === "function" ? await globalThis.caches.open(this._modelCacheName) : null; if (!useCache || !cache) { this._modelCacheState = useCache ? "unavailable" : "disabled"; } for (let shardIndex = 0; shardIndex < modelUrls.length; shardIndex += 1) { const shardUrl = modelUrls[shardIndex]; const cacheKey = normalizeAbsoluteUrl(shardUrl); let response; let headerTotal = 0; if (cache) { const cached = options.force === true ? null : await cache.match(cacheKey); if (cached) { this._modelSource = "cache"; this._modelCacheState = "hit"; this._runtimeNotes.push("model_cache_hit"); headerTotal = Number(cached.headers.get("content-length")) || 0; const resolved = headerTotal > 0 ? headerTotal : 1; shardLoaded[shardIndex] = resolved; shardTotals[shardIndex] = resolved; emitAggregateProgress(); continue; } this._modelSource = "network"; this._modelCacheState = options.force === true ? "refresh" : "miss"; } response = await this._fetchWithTimeout(shardUrl, fetchOptions, fetchTimeoutMs); if (!response.ok) { throw new Error( `Failed to prefetch model shard: ${response.status} ${response.statusText}` ); } headerTotal = Number(response.headers.get("content-length")) || 0; if (headerTotal > 0) { shardTotals[shardIndex] = headerTotal; } const putPromise = cache ? cache.put(cacheKey, response.clone()).then(() => ({ ok: true, error: null })).catch((error) => ({ ok: false, error })) : null; await drainResponseWithProgress( response, progressCallback ? (progress) => { const loaded = Number(progress?.loaded) || 0; const total = Number(progress?.total) || 0; shardLoaded[shardIndex] = loaded; if (total > 0) { shardTotals[shardIndex] = total; } emitAggregateProgress(); } : null, { chunkTimeoutMs } ); const finalLoaded = shardLoaded[shardIndex] > 0 ? shardLoaded[shardIndex] : shardTotals[shardIndex] > 0 ? shardTotals[shardIndex] : headerTotal; shardLoaded[shardIndex] = finalLoaded; if (finalLoaded > 0 && shardTotals[shardIndex] < finalLoaded) { shardTotals[shardIndex] = finalLoaded; } emitAggregateProgress(); if (putPromise) { const putResult = await putPromise; if (putResult.ok) { this._modelCacheState = "stored"; this._runtimeNotes.push("model_cache_stored"); } else { const putErrorText = String(putResult.error || "").toLowerCase(); if (putErrorText.includes("abort") || putErrorText.includes("cancel")) { throw putResult.error || new Error("Model cache prefetch was aborted."); } this._modelCacheState = "store_failed"; this._runtimeNotes.push("model_cache_store_failed"); throw new Error("Failed to store prefetched model in browser cache."); } } } if (modelUrls.length > 1) { this._runtimeNotes.push("model_split_cache_prefetched"); } return 1; } catch (error) { this._modelCacheState = "error"; const text = String(error || "").toLowerCase(); if (text.includes("abort")) { this._runtimeNotes.push("model_cache_prefetch_aborted"); } else if (text.includes("quota") || text.includes("storage")) { this._runtimeNotes.push("model_cache_quota_exceeded"); } else { this._runtimeNotes.push("model_cache_error"); } throw error; } finally { this._clearTransferAbortController(controller); } } async evictModelFromCache(url, options = {}) { this._modelCacheName = this._resolveCacheName(options); const modelUrls = expandModelShardUrls(url); if (modelUrls.length === 0) { return false; } if (!globalThis.caches || typeof globalThis.caches.open !== "function") { this._modelCacheState = "unavailable"; return false; } try { const cache = await globalThis.caches.open(this._modelCacheName); let removedCount = 0; for (const modelUrl of modelUrls) { const cacheKey = normalizeAbsoluteUrl(modelUrl); const removed = await cache.delete(cacheKey); if (removed) { removedCount += 1; } } const removedAny = removedCount > 0; this._modelCacheState = removedAny ? "evicted" : "miss"; if (removedAny) { this._runtimeNotes.push("model_cache_evicted"); } if (modelUrls.length > 1) { this._runtimeNotes.push(`model_split_cache_evicted:${removedCount}/${modelUrls.length}`); } return removedAny; } catch (_) { this._modelCacheState = "error"; this._runtimeNotes.push("model_cache_error"); return false; } } async _ensureCore() { if (this._core) { this._applyCoreLogLevel(); return this._core; } const candidates = []; if (this._config.coreModuleFactory) { candidates.push({ variant: "custom", factoryPromise: Promise.resolve(this._config.coreModuleFactory), wasmUrl: this._config.wasmUrl }); } else { if (this._preferMemory64 && typeof this._config.coreModuleUrlMem64 === "string" && this._config.coreModuleUrlMem64.length > 0) { candidates.push({ variant: "wasm64", factoryPromise: importCoreFactory(this._config.coreModuleUrlMem64), wasmUrl: this._config.wasmUrlMem64 || this._config.wasmUrl }); } candidates.push({ variant: "wasm32", factoryPromise: importCoreFactory(this._config.coreModuleUrl ?? "./llama_webgpu_core.js"), wasmUrl: this._config.wasmUrl }); } let lastError = null; for (const candidate of candidates) { if (candidate.variant === "wasm64") { this._runtimeNotes.push("core_mem64_attempt"); } try { const moduleFactory = await candidate.factoryPromise; const initTimeoutMs = this._resolveCoreInitTimeoutMs(); this._core = await Promise.race([ moduleFactory({ locateFile: (path, prefix) => { if (path.endsWith(".wasm") && candidate.wasmUrl) { return candidate.wasmUrl; } return `${prefix}${path}`; }, print: (msg) => { this._emitLogger("log", msg); }, printErr: (msg) => { const text = String(msg ?? ""); this._lastCoreErrorText = text; const trimmed = text.trim(); if (trimmed.length > 0) { const loweredTrimmed = trimmed.toLowerCase(); const isGenericAbort = loweredTrimmed === "aborted(native code called abort())" || loweredTrimmed === "native code called abort()" || loweredTrimmed === "aborted"; if (!isGenericAbort) { this._lastCoreErrorHint = trimmed; } } const classification = this._classifyCoreErrorLine(text); if (classification === "ignore") { return; } if (classification === "warmup") { if (this._logLevel >= 2) { this._suppressedWarmupWarningCount += 1; if (!this._didReportWarmupWarningSuppression) { this._didReportWarmupWarningSuppression = true; if (this._logLevel <= 2) { this._emitLogger( "log", "info: suppressing verbose warmup op logs; set bridge/runtime log level to Debug to inspect all warmup details." ); } } this._pushRuntimeNote("warmup_warning_suppressed"); return; } this._emitLogger("log", trimmed.length > 0 ? trimmed : text); return; } if (classification === "warn") { this._emitLogger("warn", trimmed.length > 0 ? trimmed : text); return; } if (classification === "info") { this._emitLogger("log", trimmed.length > 0 ? trimmed : text); return; } this._emitLogger("error", trimmed.length > 0 ? trimmed : text); }, onAbort: (reason) => { const text = String(reason ?? "").trim(); if (text.length > 0) { const token = text.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, ""); if (token.length > 0) { this._runtimeNotes.push(`core_abort:${token}`); } this._emitLogger("error", `core abort: ${text}`); } else { this._runtimeNotes.push("core_abort"); this._emitLogger("error", "core abort"); } } }), new Promise((_, reject) => { globalThis.setTimeout(() => { reject(new Error(`Bridge core init timeout (${initTimeoutMs}ms)`)); }, initTimeoutMs); }) ]); this._coreVariant = candidate.variant === "wasm64" ? "wasm64" : "wasm32"; if (candidate.variant === "wasm64") { this._runtimeNotes.push("core_mem64_active"); } else if (candidate.variant === "wasm32") { this._runtimeNotes.push("core_wasm32_active"); } break; } catch (error) { lastError = error; if (candidate.variant === "wasm64") { this._runtimeNotes.push("core_mem64_unavailable"); continue; } throw error; } } if (!this._core) { throw lastError || new Error("Failed to initialize bridge core module"); } this._applyCoreLogLevel(); return this._core; } async _probeBackends() { try { const core = await this._ensureCore(); const probeResult = Number( await core.ccall("llamadart_webgpu_probe", "number", [], [], { async: true }) ); const json = core.ccall("llamadart_webgpu_backends_json", "string", [], []); let parsed = []; try { parsed = JSON.parse(json || "[]"); } catch (_) { parsed = []; } this._backendLabels = Array.isArray(parsed) ? parsed.map((v) => String(v)) : []; this._gpuActive = probeResult === 1; } catch (_err) { this._backendLabels = []; this._gpuActive = false; } return this._gpuActive; } async loadModelFromUrl(url, options = {}, resolvedLoadState = null) { const operationSignal = options?.signal || null; const abortMessage = "Model load was cancelled."; throwIfAborted(operationSignal, abortMessage); this._abortRequested = false; this._runtimeNotes = []; this._suppressedWarmupWarningCount = 0; this._didReportWarmupWarningSuppression = false; await this._probeBackends(); throwIfAborted(operationSignal, abortMessage); const core = await this._ensureCore(); throwIfAborted(operationSignal, abortMessage); const modelUrls = expandModelShardUrls(url); if (modelUrls.length === 0) { throw new Error("Model URL is empty."); } this._releaseLoadedModel(core); this._mmProjSourceUrl = null; const configuredPoolHint = Number(this._threadPoolSizeHint); this._syncThreadPoolSizeHintFromCore(); const coreSupportsPthreads = this._coreSupportsPthreads(); this._pushRuntimeNote(`core_pthreads:${coreSupportsPthreads ? 1 : 0}`); if (!coreSupportsPthreads && Number.isFinite(configuredPoolHint) && configuredPoolHint > 1) { this._runtimeNotes.push("threads_capped_no_pthread"); } this._nCtx = Number(options.nCtx) > 0 ? Number(options.nCtx) : this._nCtx; const requestedThreads = Number(options.nThreads); if (Number.isFinite(requestedThreads) && requestedThreads > 0) { this._threads = this._capThreadsToPool(requestedThreads); } else { this._threads = this._capThreadsToPool(this._resolveAutoThreadCount()); } const requestedThreadsBatch = Number(options.nThreadsBatch); if (Number.isFinite(requestedThreadsBatch) && requestedThreadsBatch > 0) { this._threadsBatch = this._capThreadsToPool( requestedThreadsBatch, { noteTag: "threads_batch_capped_pool" } ); } else { this._threadsBatch = this._threads; } const requestedGpuLayers = Number(options.nGpuLayers); if (Number.isFinite(requestedGpuLayers)) { this._nGpuLayers = Math.trunc(requestedGpuLayers); } const isCpuModelMode = this._nGpuLayers === 0; const requestedBatch = Number(options.nBatch); this._nBatch = Number.isFinite(requestedBatch) && requestedBatch > 0 ? Math.max(32, Math.trunc(requestedBatch)) : isCpuModelMode ? Math.min(this._nCtx, 512) : 0; const requestedUbatch = Number(options.nUbatch); this._nUbatch = Number.isFinite(requestedUbatch) && requestedUbatch > 0 ? Math.max(32, Math.trunc(requestedUbatch)) : isCpuModelMode ? Math.min(this._nBatch || 256, 256) : 0; if (this._nBatch > 0 && this._nBatch > this._nCtx) { this._nBatch = this._nCtx; } if (this._nUbatch > 0 && this._nBatch > 0 && this._nUbatch > this._nBatch) { this._nUbatch = this._nBatch; } if (resolvedLoadState) { Object.assign(this, resolvedLoadState); this._threads = this._capThreadsToPool(this._threads); this._threadsBatch = this._capThreadsToPool(this._threadsBatch, { noteTag: "threads_batch_capped_pool" }); } else { this._resolveNativeLoadOptions(options); } if (Number.isFinite(this._threadPoolSizeHint) && this._threadPoolSizeHint > 0) { this._pushRuntimeNote(`thread_pool_size:${this._threadPoolSizeHint}`); } if (!isCrossOriginIsolatedRuntime()) { this._runtimeNotes.push("threads_capped_no_coi"); if (this._threads > 1) { this._threads = 1; } if (this._threadsBatch > 1) { this._threadsBatch = 1; } } this._pushRuntimeNote(`threads_batch:${this._threadsBatch}`); if (this._nBatch > 0) { this._pushRuntimeNote(`n_batch:${this._nBatch}`); } if (this._nUbatch > 0) { this._pushRuntimeNote(`n_ubatch:${this._nUbatch}`); } if (this._nSeqMax > 0) { this._pushRuntimeNote(`n_seq_max:${this._nSeqMax}`); } if (!resolvedLoadState && isCpuModelMode && !Number.isFinite(requestedBatch) && !Number.isFinite(requestedUbatch)) { this._runtimeNotes.push("cpu_batch_tuned_default"); } if (this._isSafari && this._nGpuLayers > 0) { const requestedSafariMaxLayers = Number(options.safariMaxGpuLayers); const safariMaxGpuLayers = Number.isFinite(requestedSafariMaxLayers) ? Math.max(1, Math.trunc(requestedSafariMaxLayers)) : 1; if (this._nGpuLayers > safariMaxGpuLayers) { this._nGpuLayers = safariMaxGpuLayers; this._runtimeNotes.push(`safari_gpu_layers_capped:${safariMaxGpuLayers}`); } } this._loadedModelUrl = cloneModelSource(url); if (modelUrls.length > 1) { this._runtimeNotes.push(`model_split_detected:${modelUrls.length}`); } let loadedViaRemoteFetch = false; let remoteFetchReloadUrl = null; const remoteFetchReloadChunkBytes = this._resolveRemoteFetchChunkBytes(options); if (modelUrls.length === 1) { const remoteResult = await this._tryLoadModelFromRemoteFetchBackend( core, modelUrls[0], options ); loadedViaRemoteFetch = remoteResult.loaded === true; if (loadedViaRemoteFetch) { remoteFetchReloadUrl = modelUrls[0]; } } else { this._runtimeNotes.push("model_fetch_backend_skipped_split"); } if (!loadedViaRemoteFetch) { ensureFsDirectory(core.FS, "/models"); const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null; const shardLoaded = new Array(modelUrls.length).fill(0); const shardTotals = new Array(modelUrls.length).fill(0); const emitAggregateProgress = () => { if (!progressCallback) { return; } const loaded = sumProgressValues(shardLoaded); const total = sumProgressValues(shardTotals); progressCallback({ loaded, total: total > 0 ? total : loaded }); }; const modelPaths = []; let totalModelBytes = 0; const maxStreamResumeRetries = Number.isFinite(options.streamResumeRetries) ? Math.max(0, Math.trunc(options.streamResumeRetries)) : 8; const streamChunkTimeoutMs = this._resolveStreamChunkTimeoutMs( options, 9e4 ); try { for (let shardIndex = 0; shardIndex < modelUrls.length; shardIndex += 1) { const shardUrl = modelUrls[shardIndex]; const fileName = basenameFromUrl(shardUrl); const modelPath = `/models/${fileName}`; modelPaths.push(modelPath); let shardBytes = 0; let resumeOffset = 0; let resumeAttempt = 0; let activeShardUrl = shardUrl; let knownTotalBytes = 0; while (true) { throwIfAborted(operationSignal, abortMessage); const requestHeaders = resumeOffset > 0 ? { Range: `bytes=${resumeOffset}-` } : null; const response = await this._getCachedModelResponse(activeShardUrl, { ...options, requireReadableStream: true, useCache: requestHeaders ? false : options.useCache, force: requestHeaders ? true : options.force, requestHeaders }); if (!response.ok) { throw new Error( `Failed to fetch model shard: ${response.status} ${response.statusText}` ); } const responseUrl = typeof response.url === "string" && response.url.length > 0 ? response.url : activeShardUrl; if (responseUrl !== activeShardUrl) { activeShardUrl = responseUrl; if (resumeOffset > 0) { this._runtimeNotes.push("model_stream_resume_redirect"); } } if (resumeOffset > 0 && response.status !== 206) { this._runtimeNotes.push(`model_stream_resume_status:${response.status}`); throw new Error( `Range resume not honored for model shard: ${response.status} ${response.statusText}` ); } if (!hasReadableResponseStream(response)) { this._runtimeNotes.push("model_response_nostream"); const declaredBytes = inferResponseTotalBytes(response, 0); const declaredText = declaredBytes > 0 ? `${declaredBytes} bytes` : "unknown size"; throw new Error( `Model response did not expose a readable stream (${declaredText}).` ); } this._runtimeNotes.push("model_response_stream"); const responseTotal = inferResponseTotalBytes(response, knownTotalBytes); if (responseTotal > 0) { knownTotalBytes = responseTotal; shardTotals[shardIndex] = responseTotal; } try { shardBytes = await writeResponseToFsFileWithProgress( response, core.FS, modelPath, progressCallback ? (progress) => { const loaded = Number(progress?.loaded) || 0; const total = Number(progress?.total) || 0; shardLoaded[shardIndex] = loaded; if (total > 0) { shardTotals[shardIndex] = total; } emitAggregateProgress(); } : null, { useBigIntPosition: this._coreVariant === "wasm64", startOffset: resumeOffset, preservePartialOnError: true, totalBytes: knownTotalBytes, chunkTimeoutMs: streamChunkTimeoutMs, signal: operationSignal, abortMessage } ); break; } catch (error) { const text = String(error || "").toLowerCase(); const loadedBytes = Number( error?.llamadartLoadedBytes ); if (Number.isFinite(loadedBytes) && loadedBytes >= 0) { const normalizedLoaded = Math.trunc(loadedBytes); this._runtimeNotes.push(`model_fs_write_loaded:${normalizedLoaded}`); if (normalizedLoaded > resumeOffset) { resumeOffset = normalizedLoaded; shardLoaded[shardIndex] = normalizedLoaded; if (knownTotalBytes > 0 && shardTotals[shardIndex] < knownTotalBytes) { shardTotals[shardIndex] = knownTotalBytes; } emitAggregateProgress(); } } const shouldRetryResume = isRetryableStreamNetworkError(error) && resumeOffset > 0 && resumeAttempt < maxStreamResumeRetries; if (shouldRetryResume) { resumeAttempt += 1; this._runtimeNotes.push(`model_stream_resume_retry:${resumeAttempt}`); this._runtimeNotes.push(`model_stream_resume_offset:${resumeOffset}`); continue; } if (text.includes("bigint")) { this._runtimeNotes.push("model_fs_write_bigint_error"); } else if (text.includes("abort")) { this._runtimeNotes.push("model_fs_write_abort"); } else if (text.includes("array buffer allocation failed")) { this._runtimeNotes.push("model_fs_write_arraybuffer_oom"); } else if (isRetryableStreamNetworkError(error)) { this._runtimeNotes.push("model_fs_write_network_error"); this._runtimeNotes.push("model_fs_write_failed"); } else { this._runtimeNotes.push("model_fs_write_failed"); } throw error; } } shardLoaded[shardIndex] = shardBytes; if (shardTotals[shardIndex] < shardBytes) { shardTotals[shardIndex] = shardBytes; } totalModelBytes += shardBytes; emitAggregateProgress(); } this._modelPaths = modelPaths; this._modelPath = modelPaths[0] || null; this._modelBytes = totalModelBytes; let rc = 0; let nativeLoadStarted = false; let nativeAbortCleanupDone = false; try { throwIfAborted(operationSignal, abortMessage); nativeLoadStarted = true; rc = Number( await core.ccall( "llamadart_webgpu_load_model", "number", [ "string", "number", "number", "number", "number", "number", "number", ...this._nativeLoadOptionTypes() ], [ this._modelPath, this._nCtx, this._threads, this._threadsBatch, this._nBatch, this._nUbatch, this._nGpuLayers, ...this._nativeLoadOptionValues() ], { async: true } ) ); if (operationSignal?.aborted) { try { core.ccall("llamadart_webgpu_shutdown", null, [], []); nativeAbortCleanupDone = true; } catch (_) { } throw createAbortError(abortMessage); } } catch (error) { if (operationSignal?.aborted && nativeLoadStarted && !nativeAbortCleanupDone) { try { core.ccall("llamadart_webgpu_shutdown", null, [], []); nativeAbortCleanupDone = true; } catch (_) { } error = createAbortError(abortMessage); } const text = String(error || "").toLowerCase(); if (text.includes("bigint")) { this._runtimeNotes.push("model_load_ccall_bigint_error"); } else if (text.includes("abort")) { this._runtimeNotes.push("model_load_ccall_abort"); } else { this._runtimeNotes.push("model_load_ccall_failed"); } throw error; } if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to load GGUF model", rc)); } if (modelUrls.length > 1) { this._runtimeNotes.push(`model_split_loaded:${modelUrls.length}`); } } catch (error) { this._modelBytes = 0; this._modelPath = null; this._modelPaths = []; if (operationSignal?.aborted) { this._loadedModelUrl = null; } for (const modelPath of modelPaths) { this._deleteFsFile(modelPath); } throw error; } } const shouldProbeSafariGpu = this._isSafari && this._nGpuLayers > 0 && options.safariGpuProbe !== false; if (shouldProbeSafariGpu) { if (loadedViaRemoteFetch) { this._runtimeNotes.push("safari_probe_on_fetch_backend"); } const defaultProbePrompts = [ "user: hi\nassistant:", "user: say hello in one short sentence\nassistant:" ]; const probePrompts = Array.isArray(options.safariProbePrompts) ? options.safariProbePrompts.map((v) => String(v || "").trim()).filter((v) => v.length > 0) : typeof options.safariProbePrompt === "string" && options.safariProbePrompt.trim().length > 0 ? [options.safariProbePrompt.trim()] : defaultProbePrompts; const probeTokensRaw = Number(options.safariProbeTokens); const probeTokens = Number.isFinite(probeTokensRaw) && probeTokensRaw > 0 ? Math.min(Math.trunc(probeTokensRaw), 96) : 48; const runProbe = async (probePrompt, probeSeed) => { try { const probeOutput = await this.createCompletion(probePrompt, { nPredict: probeTokens, temp: 0, topK: 1, topP: 1, penalty: 1, seed: probeSeed }); return !looksLikeCorruptedGeneration(probeOutput); } catch (_) { return false; } }; let initialProbePassed = true; for (let i = 0; i < probePrompts.length; i += 1) { const ok = await runProbe(probePrompts[i], i + 1); if (!ok) { initialProbePassed = false; break; } } if (!initialProbePassed) { this._runtimeNotes.push("safari_gpu_probe_failed"); const retryCandidates = []; if (this._nGpuLayers > 1) { retryCandidates.push(1); } retryCandidates.push(0); let stabilized = false; for (const candidateLayers of retryCandidates) { try { core.ccall("llamadart_webgpu_shutdown", null, [], []); } catch (_) { } let retryRc = 0; if (loadedViaRemoteFetch) { const reloadUrl = remoteFetchReloadUrl || modelUrls[0] || null; if (!reloadUrl) { continue; } this._runtimeNotes.push(`safari_probe_remote_fetch_retry:${candidateLayers}`); retryRc = Number( await core.ccall( "llamadart_webgpu_load_model_from_url", "number", [ "string", "number", "number", "number", "number", "number", "number", "number", ...this._nativeLoadOptionTypes() ], [ reloadUrl, this._nCtx, this._threads, this._threadsBatch, this._nBatch, this._nUbatch, candidateLayers, remoteFetchReloadChunkBytes, ...this._nativeLoadOptionValues() ], { async: true } ) ); } else { retryRc = Number( await core.ccall( "llamadart_webgpu_load_model", "number", [ "string", "number", "number", "number", "number", "number", "number", ...this._nativeLoadOptionTypes() ], [ this._modelPath, this._nCtx, this._threads, this._threadsBatch, this._nBatch, this._nUbatch, candidateLayers, ...this._nativeLoadOptionValues() ], { async: true } ) ); } if (retryRc !== 0) { continue; } this._nGpuLayers = candidateLayers; if (candidateLayers === 0) { this._runtimeNotes.push("safari_fallback_cpu"); stabilized = true; break; } let retryProbePassed = true; for (let i = 0; i < probePrompts.length; i += 1) { const ok = await runProbe(probePrompts[i], i + 11); if (!ok) { retryProbePassed = false; break; } } if (retryProbePassed) { this._runtimeNotes.push(`safari_gpu_layers_capped:${candidateLayers}`); stabilized = true; break; } } if (!stabilized) { throw new Error("Safari GPU probe failed and fallback attempts were unsuccessful."); } } else { this._runtimeNotes.push("safari_gpu_probe_passed"); } } try { const effectiveNctx = Number(core.ccall("llamadart_webgpu_get_context_size", "number", [], [])); if (effectiveNctx > 0) { this._nCtx = effectiveNctx; } } catch (_) { } this._mmProjPath = null; this._mmSupportsVision = false; this._mmSupportsAudio = false; this._mediaFileCounter = 0; this._stateFileCounter = 0; this._stagedMediaPaths = []; this._gpuActive = this._gpuActive && this._nGpuLayers > 0; if (this._releaseModelFiles() > 0) { this._runtimeNotes.push("model_file_released"); } this._emitSuppressedWarmupWarningSummaryIfNeeded(); return 1; } async loadMultimodalProjector(url) { if (!this._core || this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } if (typeof url !== "string" || url.length === 0) { throw new Error("Multimodal projector URL/path is empty."); } const core = await this._ensureCore(); ensureFsDirectory(core.FS, "/mmproj"); const fileName = basenameFromUrl(url); const mmprojPath = `/mmproj/${fileName}`; const fetchTimeoutMs = this._resolveFetchTimeoutMs({}, 18e4); const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs({}, 9e4); let lastError = null; for (let attempt = 0; attempt < 2; attempt += 1) { try { const response = await this._fetchWithTimeout( url, { cache: "no-store" }, fetchTimeoutMs ); if (!response.ok) { throw new Error( `Failed to fetch multimodal projector: ${response.status} ${response.statusText}` ); } this._mmProjPath = mmprojPath; await writeResponseToFsFileWithProgress( response, core.FS, this._mmProjPath, null, { useBigIntPosition: this._coreVariant === "wasm64", chunkTimeoutMs } ); const rc = Number( await core.ccall( "llamadart_webgpu_mmproj_load", "number", ["string"], [this._mmProjPath], { async: true } ) ); if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to load multimodal projector", rc)); } this._mmSupportsVision = Number( core.ccall("llamadart_webgpu_mmproj_supports_vision", "number", [], []) ) === 1; this._mmSupportsAudio = Number( core.ccall("llamadart_webgpu_mmproj_supports_audio", "number", [], []) ) === 1; this._mmProjSourceUrl = url; return 1; } catch (error) { lastError = error; this._mmProjPath = null; this._mmSupportsVision = false; this._mmSupportsAudio = false; this._deleteFsFile(mmprojPath); const retryable = isRetryableStreamNetworkError(error); if (!retryable || attempt >= 1) { throw error; } this._runtimeNotes.push(`mmproj_load_retry:${attempt + 1}`); } } if (lastError) { throw lastError; } throw new Error("Failed to load multimodal projector"); } async unloadMultimodalProjector() { if (!this._core) { this._mmProjPath = null; this._mmSupportsVision = false; this._mmSupportsAudio = false; this._mmProjSourceUrl = null; return; } const mmprojPath = this._mmProjPath; const rc = Number( this._core.ccall("llamadart_webgpu_mmproj_free", "number", [], []) ); if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to unload multimodal projector", rc)); } this._deleteFsFile(mmprojPath); this._mmProjPath = null; this._mmSupportsVision = false; this._mmSupportsAudio = false; this._mmProjSourceUrl = null; } // Bytes the core heap can still allocate, or null for a core without the // probe. _heapHeadroomBytes(core) { try { const value = Number(core.ccall("llamadart_webgpu_heap_headroom_bytes", "number", [], [])); return Number.isFinite(value) && value >= 0 ? value : null; } catch (_) { return null; } } // Throws when `bytes` more would not fit in the core heap, naming the // memory64 core when the wasm32 core's 4 GiB limit is the reason. _requireHeapHeadroom(core, bytes, what) { const headroom = this._heapHeadroomBytes(core); if (headroom == null || !(bytes > headroom)) { return; } const mib = (value, round) => `${round(value / (1024 * 1024))} MiB`; const detail = this._coreVariant === "wasm32" ? "the wasm32 core addresses at most 4 GiB; memory64 is required: load the bridge with coreModuleUrlMem64 and preferMemory64 in a browser with WebAssembly memory64 support" : "free memory by unloading other models or use a smaller draft model"; throw new Error( `${what} needs about ${mib(bytes, Math.ceil)} but only ${mib(headroom, Math.floor)} of WebAssembly memory is left; ${detail}.` ); } async loadDraftModel(url, options = {}) { if (!this._core || this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } if (typeof url !== "string" || url.length === 0) { throw new Error("Draft model URL is empty."); } const core = this._core; const abortMessage = "Draft model load was cancelled."; throwIfAborted(options.signal || null, abortMessage); const rc = Number(await core.ccall("llamadart_webgpu_draft_model_free", "number", [], [], { async: true })); if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to release the draft model", rc)); } this._draftModel = null; ensureFsDirectory(core.FS, "/draft"); const draftPath = `/draft/${basenameFromUrl(url)}`; const controller = this._beginTransferAbortController(); const onCallerAbort = () => controller?.abort(); options.signal?.addEventListener("abort", onCallerAbort, { once: true }); const signal = controller?.signal ?? options.signal ?? null; const targetCacheFields = [this._modelSource, this._modelCacheState, this._modelCacheName]; try { let response; try { response = await this._getCachedModelResponse(url, { useCache: options.useCache, force: options.force, signal, requireReadableStream: true }); } finally { [this._modelSource, this._modelCacheState, this._modelCacheName] = targetCacheFields; } if (!response.ok) { throw new Error(`Failed to fetch draft model: ${response.status} ${response.statusText}`); } const totalBytes = inferResponseTotalBytes(response, 0); if (totalBytes > 0) { this._requireHeapHeadroom( core, this._nGpuLayers === 0 ? totalBytes * 2 : totalBytes, "The draft model" ); } const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null; await writeResponseToFsFileWithProgress( response, core.FS, draftPath, progressCallback, { useBigIntPosition: this._coreVariant === "wasm64", totalBytes, chunkTimeoutMs: this._resolveStreamChunkTimeoutMs({}, 9e4), signal, abortMessage } ); throwIfAborted(signal, abortMessage); const loadRc = Number( await core.ccall( "llamadart_webgpu_draft_model_load", "number", ["string"], [draftPath], { async: true } ) ); if (loadRc !== 0) { let message = this._coreErrorMessage("Failed to load draft model", loadRc); if (this._coreVariant === "wasm32" && /memory|alloc/i.test(message)) { message += "; the wasm32 core addresses at most 4 GiB, so a larger draft model needs the memory64 core"; } throw new Error(message); } const info = JSON.parse( core.ccall("llamadart_webgpu_draft_model_info_json", "string", [], []) || "{}" ); this._draftModel = { url, options: { useCache: options.useCache, force: false }, info }; return { ...info }; } finally { options.signal?.removeEventListener("abort", onCallerAbort); this._clearTransferAbortController(controller); this._deleteFsFile(draftPath); } } async unloadDraftModel() { if (!this._core) { this._draftModel = null; return; } const rc = Number( await this._core.ccall("llamadart_webgpu_draft_model_free", "number", [], [], { async: true }) ); if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to unload the draft model", rc)); } this._draftModel = null; } // Writes one n-gram cache source to the core filesystem and returns its // path; the caller deletes it. async _stageNgramCache(source, name) { const core = this._core; ensureFsDirectory(core.FS, "/speculative"); this._speculativeFileCounter += 1; const path = `/speculative/${name}_${Date.now()}_${this._speculativeFileCounter}.lcs`; if (typeof source === "string") { const response = await this._fetchWithTimeout( source, { cache: "no-store" }, this._resolveFetchTimeoutMs({}, 18e4) ); if (!response.ok) { throw new Error(`Failed to fetch the ${name} n-gram cache: ${response.status} ${response.statusText}`); } core.FS.writeFile(path, new Uint8Array(await response.arrayBuffer())); } else { const bytes = source instanceof ArrayBuffer ? new Uint8Array(source) : new Uint8Array(source.buffer, source.byteOffset, source.byteLength); core.FS.writeFile(path, bytes); } return path; } // Passes the speculative settings to the core for the next // begin_generation, which consumes them. Returns the staged cache files. async _setNextSpeculative(speculative, maxTokens, stagedPaths) { const core = this._core; const staticPath = speculative.ngramCacheStatic == null ? "" : await this._stageNgramCache(speculative.ngramCacheStatic, "static"); if (staticPath) { stagedPaths.push(staticPath); } const dynamicPath = speculative.ngramCacheDynamic == null ? "" : await this._stageNgramCache(speculative.ngramCacheDynamic, "dynamic"); if (dynamicPath) { stagedPaths.push(dynamicPath); } const rc = Number(core.ccall( "llamadart_webgpu_set_next_speculative", "number", [ "string", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "string", "string", "number" ], [ speculative.strategies.join(","), speculative.draftTokenMax, speculative.draftTokenMin, speculative.minProbability, speculative.draftSplitProbability, speculative.ngramSizeN, speculative.ngramSizeM, speculative.ngramMinHits, speculative.ngramMatch, speculative.ngramTokenMin, speculative.ngramTokenMax, staticPath, dynamicPath, maxTokens ] )); if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to configure speculative decoding", rc)); } } supportsVision() { return this._mmSupportsVision; } supportsAudio() { return this._mmSupportsAudio; } getTextToSpeechCapabilities() { if (!this._core) { return { apiVersion: 0, supported: false, modelType: 0, capabilities: 0, supportsLanguage: false, supportsSpeakerReference: false, sampleRate: 0, channels: 0, reason: "WebGPU core is not initialized" }; } const raw = this._core.ccall( "llamadart_webgpu_tts_info_json", "string", [], [] ) || "{}"; try { return JSON.parse(raw); } catch (_) { return { apiVersion: 0, supported: false, modelType: 0, capabilities: 0, supportsLanguage: false, supportsSpeakerReference: false, sampleRate: 0, channels: 0, reason: "WebGPU TTS capability response is invalid" }; } } _ensureTextToSpeechDir() { try { this._core.FS.mkdir("/tts"); } catch (_) { } } async synthesizeSpeech(options = {}) { if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } if (options.signal?.aborted) { throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); } const capabilities = this.getTextToSpeechCapabilities(); if (capabilities.supported !== true) { throw new Error( capabilities.reason || "Loaded model/projector does not support text-to-speech." ); } const text = typeof options.text === "string" ? options.text : ""; if (text.length === 0) { throw new Error("Text-to-speech input text is empty."); } if (this._textToSpeechActive) { throw new Error("Text-to-speech synthesis is already active."); } this._ensureTextToSpeechDir(); const taskId = `${Date.now()}_${++this._mediaFileCounter}`; const speakerPath = `/tts/speaker_${taskId}.bin`; const outputPath = `/tts/output_${taskId}.pcm`; const speakerBytes = toUint8Array(options.speakerAudio); if (speakerBytes && speakerBytes.length > 0) { this._core.FS.writeFile(speakerPath, speakerBytes); } const promptBatchSize = Number(options.promptBatchSize) > 0 ? Math.max(1, Math.trunc(Number(options.promptBatchSize))) : 512; const maxFrames = Number(options.maxFrames) > 0 ? Math.max(1, Math.trunc(Number(options.maxFrames))) : 512; const topK = Number.isFinite(options.topK) ? Math.max(0, Math.trunc(Number(options.topK))) : 40; const topP = Number.isFinite(options.topP) ? Number(options.topP) : 0.95; const minP = Number.isFinite(options.minP) ? Number(options.minP) : 0; const temperature = Number.isFinite(options.temperature) ? Number(options.temperature) : 0.8; const seed = Number.isInteger(options.seed) ? Number(options.seed) >>> 0 : Math.floor(Math.random() * 4294967295) >>> 0; const language = typeof options.language === "string" ? options.language : ""; let started = false; this._textToSpeechActive = true; this._textToSpeechDone = new Promise((resolve) => { this._resolveTextToSpeechDone = resolve; }); try { this._abortRequested = false; const startRc = Number( await this._core.ccall( "llamadart_webgpu_tts_start", "number", ["string", "string", "string", "number", "number", "number", "number", "number", "number", "number"], [ text, language, speakerBytes && speakerBytes.length > 0 ? speakerPath : null, promptBatchSize, maxFrames, topK, topP, minP, temperature, seed ], { async: true } ) ); if (startRc !== 0) { throw new Error(this._coreErrorMessage("Failed to start text-to-speech synthesis", startRc)); } started = true; let progress = {}; for (; ; ) { if (this._abortRequested || options.signal?.aborted) { this.cancel(); } const stepRc = Number( await this._core.ccall( "llamadart_webgpu_tts_step", "number", [], [], { async: true } ) ); const rawProgress = this._core.ccall( "llamadart_webgpu_tts_progress_json", "string", [], [] ) || "{}"; progress = JSON.parse(rawProgress); options.onProgress?.(progress); if (stepRc === -6 || progress.state === 4) { throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); } if (stepRc !== 0) { throw new Error(this._coreErrorMessage("Text-to-speech synthesis failed", stepRc)); } if (progress.state === 3) { break; } await new Promise((resolve) => setTimeout(resolve, 0)); } const outputRc = Number( this._core.ccall( "llamadart_webgpu_tts_write_pcm", "number", ["string"], [outputPath] ) ); if (outputRc !== 0) { throw new Error(this._coreErrorMessage("Failed to read synthesized speech", outputRc)); } const bytes = this._core.FS.readFile(outputPath); if (!(bytes instanceof Uint8Array) || bytes.byteLength === 0 || bytes.byteLength % 4 !== 0) { throw new Error("Synthesized PCM output is empty or malformed."); } const copy = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); const pcm = new Float32Array(copy); return { pcm, sampleRate: Number(capabilities.sampleRate) || 0, channels: Number(capabilities.channels) || 1, sampleCount: pcm.length, framesGenerated: Number(progress.framesGenerated) || 0, truncated: progress.truncated === true }; } finally { if (started) { try { this._core.ccall("llamadart_webgpu_tts_reset", "number", [], []); } catch (_) { } } this._deleteFsFile(speakerPath); this._deleteFsFile(outputPath); this._textToSpeechActive = false; this._resolveTextToSpeechDone?.(); this._resolveTextToSpeechDone = null; this._textToSpeechDone = null; } } getDecisionCapabilities() { const unsupported = (reason) => ({ apiVersion: DECISION_API_VERSION, supported: false, reason }); const core = this._core; if (!core) { return unsupported("WebGPU core is not initialized"); } if (typeof core._llamadart_webgpu_decision_capabilities_json !== "function") { return unsupported("This WebGPU core build does not include decision heads."); } const coreVersion = Number( core.ccall("llamadart_webgpu_decision_api_version", "number", [], []) ); if (coreVersion !== DECISION_API_VERSION) { return unsupported( `The WebGPU core implements decision API version ${coreVersion}; this bridge needs version ${DECISION_API_VERSION}.` ); } const raw = core.ccall( "llamadart_webgpu_decision_capabilities_json", "string", [], [] ) || "{}"; try { return JSON.parse(raw); } catch (_) { return unsupported("WebGPU decision capability response is invalid"); } } _ensureDecisionDir() { try { this._core.FS.mkdir("/decision"); } catch (_) { } } async _stageDecisionHeadFromUrl(url, headPath, onProgress) { const fetchTimeoutMs = this._resolveFetchTimeoutMs({}, 18e4); const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs({}, 9e4); for (let attempt = 0; ; attempt += 1) { try { const response = await this._fetchWithTimeout( url, { cache: "no-store" }, fetchTimeoutMs ); if (!response.ok) { throw new Error( `Failed to fetch decision head: ${response.status} ${response.statusText}` ); } await writeResponseToFsFileWithProgress( response, this._core.FS, headPath, typeof onProgress === "function" ? onProgress : null, { useBigIntPosition: this._coreVariant === "wasm64", chunkTimeoutMs } ); return; } catch (error) { this._unlinkDecisionFile(headPath); if (!isRetryableStreamNetworkError(error) || attempt >= 1) { throw error; } this._runtimeNotes.push(`decision_head_fetch_retry:${attempt + 1}`); } } } async loadDecisionHead(source, options = {}) { if (!this._core || this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } const configJson = options?.configJson ?? null; if (configJson !== null && typeof configJson !== "string") { throw new TypeError("Decision head configJson must be a string."); } let bytes = null; if (typeof source === "string") { if (source.length === 0) { throw new Error("Decision head URL is empty."); } } else { bytes = bufferSourceBytes(source); if (!bytes) { throw new TypeError("Decision head source must be a URL string, an ArrayBuffer or a typed array."); } if (bytes.byteLength === 0) { throw new Error("Decision head bytes are empty."); } } const capabilities = this.getDecisionCapabilities(); if (capabilities.supported !== true) { throw new Error( capabilities.reason || "The loaded model does not support decision heads." ); } const core = this._core; this._ensureDecisionDir(); const taskId = `${Date.now()}_${++this._decisionFileCounter}`; const headPath = `/decision/head_${taskId}.safetensors`; const configPath = configJson === null ? null : `/decision/config_${taskId}.json`; try { let label = "decision head bytes"; if (bytes) { core.FS.writeFile(headPath, bytes); } else { label = basenameFromUrl(source).slice(0, 256); await this._stageDecisionHeadFromUrl(source, headPath, options?.onProgress); } if (configPath !== null) { core.FS.writeFile(configPath, textEncoder.encode(configJson)); } const handle = Number( await core.ccall( "llamadart_webgpu_decision_load", "number", ["string", "string", "string"], [headPath, label, configPath], { async: true } ) ); if (handle <= 0) { throw new Error(this._coreErrorMessage("Failed to load decision head", handle)); } const raw = core.ccall( "llamadart_webgpu_decision_head_info_json", "string", ["number"], [handle] ) || "{}"; return JSON.parse(raw); } finally { this._unlinkDecisionFile(headPath); this._unlinkDecisionFile(configPath); } } async runDecision(handle, sequences) { const nativeHandle = decisionHandleFrom(handle); const input = encodeDecisionSequences(sequences); if (!this._core || this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } const core = this._core; this._ensureDecisionDir(); const taskId = `${Date.now()}_${++this._decisionFileCounter}`; const inputPath = `/decision/input_${taskId}.bin`; const outputPath = `/decision/output_${taskId}.bin`; try { core.FS.writeFile(inputPath, input); const count = Number( await core.ccall( "llamadart_webgpu_decision_run", "number", ["number", "string", "string"], [nativeHandle, inputPath, outputPath], { async: true } ) ); if (count < 0) { throw new Error(this._coreErrorMessage("Decision run failed", count)); } return decodeDecisionOutputs(core.FS.readFile(outputPath), count); } finally { this._unlinkDecisionFile(inputPath); this._unlinkDecisionFile(outputPath); } } async freeDecisionHead(handle) { const nativeHandle = decisionHandleFrom(handle); if (!this._core) { return; } await this._core.ccall( "llamadart_webgpu_decision_free", "number", ["number"], [nativeHandle], { async: true } ); } getLoraAdapterCapabilities() { const unsupported = (reason) => ({ apiVersion: LORA_API_VERSION, supported: false, reason }); const core = this._core; if (!core) { return unsupported("WebGPU core is not initialized"); } if (typeof core._llamadart_webgpu_lora_api_version !== "function") { return unsupported("This WebGPU core build does not include LoRA adapters."); } const coreVersion = Number( core.ccall("llamadart_webgpu_lora_api_version", "number", [], []) ); if (coreVersion !== LORA_API_VERSION) { return unsupported( `The WebGPU core implements LoRA API version ${coreVersion}; this bridge needs version ${LORA_API_VERSION}.` ); } return { apiVersion: LORA_API_VERSION, supported: true }; } _requireLoraCore() { if (!this._core || this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } const capabilities = this.getLoraAdapterCapabilities(); if (capabilities.supported !== true) { throw new Error(capabilities.reason || "This bridge cannot load LoRA adapters."); } return this._core; } /** * Downloads a URL adapter into `adapterPath`, through the Cache API unless * `options.useCache` is false. A cache miss stores the response while it is * written, so progress starts with the first chunk. */ async _stageLoraAdapterFromUrl(url, adapterPath, options) { const core = this._core; const signal = options.signal || null; const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null; const fetchTimeoutMs = this._resolveFetchTimeoutMs(options, 18e4); const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs(options, 9e4); const fetchInit = signal ? { signal } : {}; const cacheKey = normalizeAbsoluteUrl(url); let cache = null; if (options.useCache !== false && typeof globalThis.caches?.open === "function") { try { cache = await globalThis.caches.open(this._resolveCacheName(options)); } catch (_) { this._runtimeNotes.push("lora_cache_error"); } } for (let attempt = 0; ; attempt += 1) { throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE); try { let response = null; if (cache) { try { response = await cache.match(cacheKey) || null; } catch (_) { this._runtimeNotes.push("lora_cache_error"); } } let stored = null; if (!response) { response = await this._fetchWithTimeout( url, cache ? fetchInit : { cache: "no-store", ...fetchInit }, fetchTimeoutMs ); if (!response.ok) { throw new Error( `Failed to fetch LoRA adapter: ${response.status} ${response.statusText}` ); } if (cache) { stored = cache.put(cacheKey, response.clone()).then( () => { this._runtimeNotes.push("lora_cache_stored"); }, () => { this._runtimeNotes.push("lora_cache_store_failed"); } ); } } await writeResponseToFsFileWithProgress( response, core.FS, adapterPath, progressCallback, { useBigIntPosition: this._coreVariant === "wasm64", chunkTimeoutMs, signal, abortMessage: LORA_LOAD_ABORT_MESSAGE } ); await stored; return; } catch (error) { this._deleteFsFile(adapterPath); throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE); if (!isRetryableStreamNetworkError(error) || attempt >= 1) { throw error; } this._runtimeNotes.push(`lora_fetch_retry:${attempt + 1}`); } } } async loadLoraAdapter(source, options = {}) { let bytes = null; if (typeof source === "string") { if (source.length === 0) { throw new Error("LoRA adapter URL is empty."); } } else { bytes = bufferSourceBytes(source); if (!bytes) { throw new TypeError("LoRA adapter source must be a URL string, an ArrayBuffer or a typed array."); } if (bytes.byteLength === 0) { throw new Error("LoRA adapter bytes are empty."); } } const core = this._requireLoraCore(); const signal = options.signal || null; throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE); ensureFsDirectory(core.FS, "/lora"); const adapterPath = `/lora/adapter_${++this._loraFileCounter}.gguf`; try { if (bytes) { core.FS.writeFile(adapterPath, bytes); } else { await this._stageLoraAdapterFromUrl(source, adapterPath, options); } throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE); const handle = Number( await core.ccall( "llamadart_webgpu_lora_load", "number", ["string"], [adapterPath], { async: true } ) ); if (handle <= 0) { throw new Error(this._coreErrorMessage("Failed to load LoRA adapter", handle)); } this._loraAdaptersLoaded = true; return { handle }; } finally { this._deleteFsFile(adapterPath); } } async _callLoraCore(name, argTypes, args, failure) { const core = this._requireLoraCore(); const rc = Number(await core.ccall(name, "number", argTypes, args, { async: true })); if (rc !== 0) { throw new Error(this._coreErrorMessage(failure, rc)); } } async setLoraAdapter(handle, scale = 1) { await this._callLoraCore( "llamadart_webgpu_lora_set", ["number", "number"], [loraHandleFrom(handle), loraScaleFrom(scale)], "Failed to apply LoRA adapter" ); } async removeLoraAdapter(handle) { await this._callLoraCore( "llamadart_webgpu_lora_remove", ["number"], [loraHandleFrom(handle)], "Failed to remove LoRA adapter" ); } async clearLoraAdapters() { await this._callLoraCore( "llamadart_webgpu_lora_clear", [], [], "Failed to clear LoRA adapters" ); } _clearStagedMediaFiles() { if (!this._core || this._stagedMediaPaths.length === 0) { this._stagedMediaPaths = []; return; } for (const mediaPath of this._stagedMediaPaths) { try { this._core.FS.unlink(mediaPath); } catch (_) { } } this._stagedMediaPaths = []; } _clearPendingMedia() { this._core?.ccall("llamadart_webgpu_media_clear_pending", null, [], []); this._clearStagedMediaFiles(); } _persistMediaBytes(bytes, extension = ".bin") { if (!this._core) { throw new Error("WebGPU core is not initialized."); } try { this._core.FS.mkdir("/media"); } catch (_) { } this._mediaFileCounter += 1; const suffix = typeof extension === "string" && extension.startsWith(".") ? extension : ".bin"; const mediaPath = `/media/input_${Date.now()}_${this._mediaFileCounter}${suffix}`; this._core.FS.writeFile(mediaPath, bytes); this._stagedMediaPaths.push(mediaPath); return mediaPath; } _addMediaFile(mediaPath) { const rc = Number( this._core.ccall( "llamadart_webgpu_media_add_file", "number", ["string"], [mediaPath] ) ); if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to add media file", rc)); } } _addRawRgbMediaBytes(bytes, width, height) { const useHeapBuffer = this._core && typeof this._core._malloc === "function" && typeof this._core._free === "function" && this._core.HEAPU8 && typeof this._core.HEAPU8.set === "function"; let rc = 0; if (useHeapBuffer) { const ptr = this._core._malloc(bytes.length); if (!Number.isFinite(ptr) || ptr <= 0) { throw new Error("Failed to allocate core heap buffer for RGB media bytes"); } try { this._core.HEAPU8.set(bytes, ptr); rc = Number( this._core.ccall( "llamadart_webgpu_media_add_rgb", "number", ["number", "number", "number", "number"], [width, height, ptr, bytes.length] ) ); } finally { this._core._free(ptr); } } else { rc = Number( this._core.ccall( "llamadart_webgpu_media_add_rgb", "number", ["number", "number", "array", "number"], [width, height, bytes, bytes.length] ) ); } if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to add raw RGB media bytes", rc)); } } _addAudioSamples(samples) { const sampleBytes = new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength); const useHeapBuffer = this._core && typeof this._core._malloc === "function" && typeof this._core._free === "function" && this._core.HEAPU8 && typeof this._core.HEAPU8.set === "function"; let rc = 0; if (useHeapBuffer) { const ptr = this._core._malloc(sampleBytes.length); if (!Number.isFinite(ptr) || ptr <= 0) { throw new Error("Failed to allocate core heap buffer for audio samples"); } try { this._core.HEAPU8.set(sampleBytes, ptr); rc = Number( this._core.ccall( "llamadart_webgpu_media_add_audio_f32", "number", ["number", "number"], [ptr, samples.length] ) ); } finally { this._core._free(ptr); } } else { rc = Number( this._core.ccall( "llamadart_webgpu_media_add_audio_f32", "number", ["array", "number"], [sampleBytes, samples.length] ) ); } if (rc !== 0) { throw new Error(this._coreErrorMessage("Failed to add audio samples", rc)); } } async _fetchMediaBytes(url) { const response = await this._fetchWithTimeout( url, { cache: "no-store" }, this._resolveFetchTimeoutMs({}, 12e4) ); if (!response.ok) { throw new Error(`Failed to fetch media: ${response.status} ${response.statusText}`); } return new Uint8Array(await response.arrayBuffer()); } async _prepareImageBytesForMultimodal(bytes, options = {}) { const maxPixels = this._resolveMediaImageMaxPixels(options); const maxEdge = this._resolveMediaImageMaxEdge(options); if (maxPixels <= 0 && maxEdge <= 0) { return null; } return decodeImageBytesToRgb(bytes, { maxPixels, maxEdge }); } async _stageMultimodalParts(parts, options = {}) { this._clearPendingMedia(); const mediaParts = Array.isArray(parts) ? parts : []; if (mediaParts.length === 0) { return; } if (!this._mmProjPath) { throw new Error( "Multimodal input requires a loaded projector. Call loadMultimodalProjector first." ); } for (const rawPart of mediaParts) { const part = rawPart && typeof rawPart === "object" ? rawPart : {}; const type = String(part.type || "").toLowerCase(); if (type === "image") { const bytes = toUint8Array(part.bytes); if (bytes && bytes.length > 0) { const width = Number(part.width); const height = Number(part.height); const isRawRgb = Number.isInteger(width) && Number.isInteger(height) && width > 0 && height > 0 && bytes.length === width * height * 3; if (isRawRgb) { this._addRawRgbMediaBytes(bytes, width, height); } else { const prepared2 = await this._prepareImageBytesForMultimodal(bytes, options); if (prepared2 && prepared2.bytes && prepared2.bytes.length > 0) { const mediaPath = this._persistMediaBytes(prepared2.bytes, ".img"); this._addMediaFile(mediaPath); if (prepared2.resized) { this._runtimeNotes.push( `media_image_resized:${prepared2.sourceWidth}x${prepared2.sourceHeight}->${prepared2.width}x${prepared2.height}` ); } } else { const mediaPath = this._persistMediaBytes(bytes, ".img"); this._addMediaFile(mediaPath); } } continue; } if (typeof part.url !== "string" || part.url.length === 0) { throw new Error("Image part must provide bytes or url."); } const fetched = await this._fetchMediaBytes(part.url); const prepared = await this._prepareImageBytesForMultimodal(fetched, options); if (prepared && prepared.bytes && prepared.bytes.length > 0) { const mediaPath = this._persistMediaBytes(prepared.bytes, ".img"); this._addMediaFile(mediaPath); if (prepared.resized) { this._runtimeNotes.push( `media_image_resized:${prepared.sourceWidth}x${prepared.sourceHeight}->${prepared.width}x${prepared.height}` ); } } else { const mediaPath = this._persistMediaBytes(fetched, ".img"); this._addMediaFile(mediaPath); } continue; } if (type === "audio") { const samples = toFloat32Array(part.samples); if (samples && samples.length > 0) { this._addAudioSamples(samples); continue; } const bytes = toUint8Array(part.bytes); if (bytes && bytes.length > 0) { const mediaPath2 = this._persistMediaBytes(bytes, ".aud"); this._addMediaFile(mediaPath2); continue; } if (typeof part.url !== "string" || part.url.length === 0) { throw new Error("Audio part must provide samples, bytes, or url."); } const fetched = await this._fetchMediaBytes(part.url); const mediaPath = this._persistMediaBytes(fetched, ".aud"); this._addMediaFile(mediaPath); } } } getCompletionCapabilities() { const core = this._core; if (!core || typeof core._llamadart_webgpu_completion_capabilities_json !== "function") { return { ...NO_COMPLETION_CAPABILITIES }; } return completionCapabilitiesFrom( core.ccall("llamadart_webgpu_completion_capabilities_json", "string", [], []) ); } async createCompletion(prompt, options = {}) { if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } const sampling = resolveCompletionSamplingOptions(options); requireCompletionCapabilities(sampling, this.getCompletionCapabilities()); this._abortRequested = false; const startedAt = performance.now(); let firstTokenAt = null; let nPredict = Number(options.nPredict) > 0 ? Number(options.nPredict) : 256; const hasMediaParts = Array.isArray(options.parts) && options.parts.length > 0; if (hasMediaParts) { const requestedMediaCap = Number(options.mediaMaxPredict); const mediaCap = Number.isFinite(requestedMediaCap) && requestedMediaCap > 0 ? Math.max(32, Math.trunc(requestedMediaCap)) : 256; if (nPredict > mediaCap) { nPredict = mediaCap; this._runtimeNotes.push(`media_n_predict_capped:${mediaCap}`); } } const temp = Number.isFinite(options.temp) ? Number(options.temp) : 0.8; const topK = Number.isFinite(options.topK) ? Number(options.topK) : 40; const topP = Number.isFinite(options.topP) ? Number(options.topP) : 0.95; const penalty = Number.isFinite(options.penalty) ? Number(options.penalty) : 1.1; const grammar = typeof options.grammar === "string" && options.grammar.length > 0 ? options.grammar : null; const seed = Number.isInteger(options.seed) ? Number(options.seed) : Math.floor(Math.random() * 4294967295); await this._stageMultimodalParts(options.parts, options); let generationStarted = false; const speculativePaths = []; try { if (sampling.speculative) { await this._setNextSpeculative(sampling.speculative, nPredict, speculativePaths); } const beginRc = Number( await this._core.ccall( "llamadart_webgpu_begin_generation", "number", [ "string", "number", "number", "number", "number", "string", "number", "number", "number", "number", "string", "string", "string" ], [ String(prompt), temp, topK, topP, penalty, grammar, seed >>> 0, sampling.minP, sampling.presencePenalty, sampling.thinkingBudget?.maxTokens ?? 0, sampling.thinkingBudget?.startTag ?? null, sampling.thinkingBudget?.endTag ?? null, sampling.thinkingBudget?.forcedMessage ?? null ], { async: true } ) ); if (beginRc === GENERATION_ALREADY_ACTIVE_RC) { throw new Error(GENERATION_ALREADY_ACTIVE_MESSAGE); } if (beginRc !== 0) { throw new Error(this._coreErrorMessage("Failed to start generation", beginRc)); } generationStarted = true; for (const path of speculativePaths.splice(0)) { this._deleteFsFile(path); } let generated = 0; const shouldEmitCurrentText = options.emitCurrentTextOnToken !== false; const tokenEventEncoding = typeof options.tokenEventEncoding === "string" ? String(options.tokenEventEncoding || "").toLowerCase() : "bytes"; const emitTokenText = tokenEventEncoding === "text"; const yieldInterval = 4; let streamed = ""; let emittedStableText = ""; let finishReason = "length"; while (generated < nPredict) { if (this._abortRequested || options.signal?.aborted) { finishReason = "cancelled"; break; } const stepRc = Number( await this._core.ccall( "llamadart_webgpu_next_token", "number", [], [], { async: true } ) ); if (stepRc === 0) { finishReason = this._abortRequested || options.signal?.aborted ? "cancelled" : "stop"; break; } if (stepRc < 0) { const stepErrorText = this._coreErrorMessage("Generation step failed", stepRc); if (generated > 0 && this._isContextLimitGenerationError(stepErrorText)) { this._runtimeNotes.push("generation_stopped_context_limit"); this._emitLogger( "warn", "warning: generation reached context/memory limit; returning partial output." ); break; } if (sampling.speculative == null && this._shouldAttemptGenerationRecovery(stepErrorText, options, generated)) { const recovered = await this._recoverGenerationWithCpuFallback(options); if (recovered) { if (generationStarted) { try { this._core.ccall("llamadart_webgpu_end_generation", null, [], []); } catch (_) { } generationStarted = false; } this._clearPendingMedia(); const retryOptions = { ...options, _llamadartGenerationRecoveryAttempted: true }; return await this.createCompletion(prompt, retryOptions); } } throw new Error(stepErrorText); } generated += 1; const fullText = this._core.ccall("llamadart_webgpu_last_output", "string", [], []) || ""; streamed = fullText; const stableText = trimUnstableUtf8Tail(fullText); if (!stableText.startsWith(emittedStableText)) { emittedStableText = ""; } const deltaText = stableText.slice(emittedStableText.length); if (deltaText.length === 0) { continue; } emittedStableText = stableText; firstTokenAt ??= performance.now(); if (typeof options.onToken === "function") { const piecePayload = emitTokenText ? deltaText : textEncoder.encode(deltaText); options.onToken(piecePayload, shouldEmitCurrentText ? fullText : null); } if (yieldInterval > 0 && generated % yieldInterval === 0) { await new Promise((resolve) => setTimeout(resolve, 0)); } } const text = this._core.ccall("llamadart_webgpu_last_output", "string", [], []) || streamed || ""; const tailText = text.startsWith(emittedStableText) ? text.slice(emittedStableText.length) : ""; if (tailText.length > 0) { firstTokenAt ??= performance.now(); if (typeof options.onToken === "function") { const piecePayload = emitTokenText ? tailText : textEncoder.encode(tailText); options.onToken(piecePayload, shouldEmitCurrentText ? text : null); } } if (typeof options.onUsage === "function") { const counts = JSON.parse( this._core.ccall("llamadart_webgpu_last_generation_usage_json", "string", [], []) || "{}" ); const speculative = counts?.speculative; options.onUsage({ promptTokens: Number(counts?.promptTokens) || 0, cachedPromptTokens: Number(counts?.cachedPromptTokens) || 0, completionTokens: Number(counts?.completionTokens) || 0, timeToFirstTokenMs: firstTokenAt == null ? null : firstTokenAt - startedAt, durationMs: performance.now() - startedAt, finishReason, ...speculative && typeof speculative === "object" ? { speculative: { draftTokens: Number(speculative.draftTokens) || 0, acceptedDraftTokens: Number(speculative.acceptedDraftTokens) || 0, draftAttempts: Number(speculative.draftAttempts) || 0, verifyTokens: Number(speculative.verifyTokens) || 0, replayTokens: Number(speculative.replayTokens) || 0 } } : {} }); } return text; } finally { if (generationStarted) { await this._core.ccall("llamadart_webgpu_end_generation", null, [], [], { async: true }); } for (const path of speculativePaths) { this._deleteFsFile(path); } this._clearPendingMedia(); } } async tokenize(text, _addSpecial = true) { if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } const rc = Number( await this._core.ccall( "llamadart_webgpu_tokenize_to_json", "number", ["string", "number"], [String(text), _addSpecial ? 1 : 0], { async: true } ) ); if (rc < 0) { throw new Error(this._coreErrorMessage("Tokenization failed", rc)); } const raw = this._core.ccall("llamadart_webgpu_last_tokens_json", "string", [], []) || "[]"; const parsed = JSON.parse(raw); return Array.isArray(parsed) ? parsed.map((v) => Number(v) | 0) : []; } _ensureStateDir() { const core = this._core; if (!core?.FS) { throw new Error("Bridge filesystem is not initialized"); } ensureFsDirectory(core.FS, "/states"); } _normalizeStateTokens(tokens) { const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); return normalized.map((value) => Number(value) | 0); } _nextStateTempPath() { this._ensureStateDir(); this._stateFileCounter += 1; return `/states/state_${Date.now()}_${this._stateFileCounter}.bin`; } async stateSaveFile(path, tokens = []) { if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } if (typeof path !== "string" || path.length === 0) { throw new Error("State file path is empty."); } const normalized = this._normalizeStateTokens(tokens); const tokenText = JSON.stringify(normalized); const rc = Number( await this._core.ccall( "llamadart_webgpu_state_save_file", "number", ["string", "string"], [path, tokenText], { async: true } ) ); if (rc < 0) { throw new Error(this._coreErrorMessage("State save failed", rc)); } return true; } async stateLoadFile(path, tokenCapacity = this.getContextSize()) { if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } if (typeof path !== "string" || path.length === 0) { throw new Error("State file path is empty."); } const capacity = Number(tokenCapacity) > 0 ? Math.trunc(Number(tokenCapacity)) : this.getContextSize(); const rc = Number( await this._core.ccall( "llamadart_webgpu_state_load_file", "number", ["string", "number"], [path, capacity], { async: true } ) ); if (rc < 0) { throw new Error(this._coreErrorMessage("State load failed", rc)); } const raw = this._core.ccall("llamadart_webgpu_last_tokens_json", "string", [], []) || "[]"; const parsed = JSON.parse(raw); const restoredTokens = Array.isArray(parsed) ? parsed.map((v) => Number(v) | 0) : []; return { tokens: restoredTokens }; } async stateSaveBytes(tokens = []) { if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } const tempPath = this._nextStateTempPath(); try { await this.stateSaveFile(tempPath, tokens); const bytes = this._core.FS.readFile(tempPath); return bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes || []); } finally { this._deleteFsFile(tempPath); } } async stateLoadBytes(bytes, tokenCapacity = this.getContextSize()) { if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } const normalizedBytes = toUint8Array(bytes); if (!normalizedBytes || normalizedBytes.length === 0) { throw new Error("State bytes are empty."); } const tempPath = this._nextStateTempPath(); try { this._core.FS.writeFile(tempPath, normalizedBytes); return await this.stateLoadFile(tempPath, tokenCapacity); } finally { this._deleteFsFile(tempPath); } } async detokenize(tokens, _special = false) { if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); const tokenText = JSON.stringify(normalized.map((v) => Number(v) | 0)); const rc = Number( await this._core.ccall( "llamadart_webgpu_detokenize_from_json", "number", ["string", "number"], [tokenText, _special ? 1 : 0], { async: true } ) ); if (rc < 0) { throw new Error(this._coreErrorMessage("Detokenization failed", rc)); } return this._core.ccall("llamadart_webgpu_last_detokenized", "string", [], []) || ""; } _embeddingAssetVersion() { return embeddingAssetVersion(this._coreVariant === "wasm64" ? this._config.coreModuleUrlMem64 : this._config.coreModuleUrl); } async getEmbeddingCapabilities() { if (this._modelBytes <= 0) return { ...EMPTY_EMBEDDING_CAPABILITIES }; if (typeof this._core?._llamadart_webgpu_embedding_capabilities_json !== "function" || typeof this._core?._llamadart_webgpu_embed_parts_to_json !== "function") { throw missingEmbeddingCapability(this._embeddingAssetVersion()); } return embeddingCapabilitiesFrom( this._core.ccall("llamadart_webgpu_embedding_capabilities_json", "string", [], []), this._embeddingAssetVersion() ); } async _preflightEmbeddingInputs(inputs) { if (!inputs.some(Array.isArray)) return; const caps = await this.getEmbeddingCapabilities(); for (const input of inputs) requireEmbeddingModalities(input, caps, !!this._mmProjPath); } async _embedParts(parts, options) { const prompt = parts.map((part) => part.type === "text" ? part.text : EMBEDDING_MEDIA_MARKER).join(""); const promptBytes = textEncoder.encode(prompt); const media = parts.filter((part) => part.type !== "text"); try { await this._stageMultimodalParts(media); if (this._abortRequested) throw createAbortError("Embedding was cancelled."); const rc = Number(await this._core.ccall( "llamadart_webgpu_embed_parts_to_json", "number", ["array", "number", "number"], [promptBytes, options?.normalize !== false ? 1 : 0, promptBytes.length], { async: true } )); if (this._abortRequested) throw createAbortError("Embedding was cancelled."); if (rc < 0) throw embeddingCoreError(rc, this._embeddingAssetVersion()); const raw = this._core.ccall("llamadart_webgpu_last_embedding_json", "string", [], []) || "[]"; const parsed = JSON.parse(raw); return Array.isArray(parsed) ? parsed.map((v) => Number.isFinite(Number(v)) ? Number(v) : 0) : []; } catch (error) { if (error instanceof WebAssembly.RuntimeError) throw error; if (error instanceof EmbeddingError || error?.name === "AbortError") throw error; throw new EmbeddingError("EMBEDDING_FAILED", "Embedding media preparation or execution failed."); } finally { this._clearPendingMedia(); } } async embed(text, options = {}) { if (Array.isArray(text)) { const parts = snapshotEmbedInput(text); this._abortRequested = false; await this._preflightEmbeddingInputs([parts]); if (this._abortRequested) throw createAbortError("Embedding was cancelled."); return this._embedParts(parts, options); } if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } const normalize = options?.normalize !== false; const rc = Number( await this._core.ccall( "llamadart_webgpu_embed_to_json", "number", ["string", "number"], [String(text), normalize ? 1 : 0], { async: true } ) ); if (rc < 0) { throw new Error(this._coreErrorMessage("Embedding generation failed", rc)); } const raw = this._core.ccall("llamadart_webgpu_last_embedding_json", "string", [], []) || "[]"; const parsed = JSON.parse(raw); return Array.isArray(parsed) ? parsed.map((v) => { const numeric = Number(v); return Number.isFinite(numeric) ? numeric : 0; }) : []; } async embedBatch(texts, options = {}) { const normalized = Array.isArray(texts) ? texts : Array.from(texts || []); if (normalized.length === 0) { return []; } const inputs = normalized.map((input) => typeof input === "string" ? input : snapshotEmbedInput(input)); this._abortRequested = false; await this._preflightEmbeddingInputs(inputs); const normalize = options?.normalize !== false; const vectors = []; for (const text of inputs) { if (this._abortRequested) throw createAbortError("Batch embedding was cancelled."); vectors.push(await (typeof text === "string" ? this.embed(text, { normalize }) : this._embedParts(text, { normalize }))); } return vectors; } async scoreNextToken(prompt, options = {}) { if (this._modelBytes <= 0) { throw new Error("No model loaded. Call loadModelFromUrl first."); } if (typeof prompt !== "string") { throw new TypeError("Next-token scoring prompt must be a string."); } const candidates = nextTokenCandidateIds(options?.candidates); const topK = options?.topK ?? 0; if (!isInt32(topK)) { throw new TypeError(`Next-token topK is ${String(topK)}; expected a 32-bit integer.`); } const reusePromptPrefix = options?.reusePromptPrefix !== false; const rc = Number( await this._core.ccall( "llamadart_webgpu_score_next_token_to_json", "number", ["string", "string", "number", "number"], [prompt, JSON.stringify(candidates), topK, reusePromptPrefix ? 1 : 0], { async: true } ) ); if (rc < 0) { throw new Error(this._coreErrorMessage("Next-token scoring failed", rc)); } const raw = this._core.ccall("llamadart_webgpu_last_next_token_scores_json", "string", [], []) || "{}"; const parsed = JSON.parse(raw); return { candidates: scoredTokensFrom(parsed?.candidates), top: scoredTokensFrom(parsed?.top), promptTokens: Number(parsed?.promptTokens) || 0 }; } getModelMetadata() { let modelMetadata = {}; try { const raw = this._core?.ccall("llamadart_webgpu_model_meta_json", "string", [], []); if (raw) { const parsed = JSON.parse(raw); if (parsed && typeof parsed === "object") { modelMetadata = parsed; } } } catch (_) { } return { ...modelMetadata, "llamadart.webgpu.prototype": "1", "llamadart.webgpu.backends": this._backendLabels.join(","), "llamadart.webgpu.model_bytes": String(this._modelBytes), "llamadart.webgpu.n_threads": String(this._threads), "llamadart.webgpu.n_threads_batch": String(this._threadsBatch), "llamadart.webgpu.n_batch": this._nBatch > 0 ? String(this._nBatch) : "", "llamadart.webgpu.n_ubatch": this._nUbatch > 0 ? String(this._nUbatch) : "", "llamadart.webgpu.n_seq_max": this._nSeqMax > 0 ? String(this._nSeqMax) : "", "llamadart.webgpu.flash_attention": String(this._flashAttention), "llamadart.webgpu.cache_type_k": String(this._cacheTypeK), "llamadart.webgpu.cache_type_v": String(this._cacheTypeV), "llamadart.webgpu.kv_unified": this._kvUnified >= 0 ? String(this._kvUnified) : "", "llamadart.webgpu.rope_freq_base": this._ropeFrequencyBase > 0 ? String(this._ropeFrequencyBase) : "", "llamadart.webgpu.rope_freq_scale": this._ropeFrequencyScale > 0 ? String(this._ropeFrequencyScale) : "", "llamadart.webgpu.split_mode": this._splitMode >= 0 ? String(this._splitMode) : "", "llamadart.webgpu.main_gpu": this._mainGpu >= 0 ? String(this._mainGpu) : "", "llamadart.webgpu.thread_pool_size": Number.isFinite(this._threadPoolSizeHint) && this._threadPoolSizeHint > 0 ? String(this._threadPoolSizeHint) : "", "llamadart.webgpu.n_gpu_layers": String(this._nGpuLayers), "llamadart.webgpu.core_variant": this._coreVariant, "llamadart.webgpu.model_source": this._modelSource, "llamadart.webgpu.model_cache_state": this._modelCacheState, "llamadart.webgpu.model_cache_name": this._modelCacheName, "llamadart.webgpu.runtime_notes": this._runtimeNotes.join(";"), "llamadart.webgpu.mmproj_loaded": this._mmProjPath ? "1" : "0", "llamadart.webgpu.supports_vision": this._mmSupportsVision ? "1" : "0", "llamadart.webgpu.supports_audio": this._mmSupportsAudio ? "1" : "0" }; } getContextSize() { try { const nctx = Number(this._core?.ccall("llamadart_webgpu_get_context_size", "number", [], [])); if (nctx > 0) { return nctx; } } catch (_) { } return this._nCtx; } isGpuActive() { return this._gpuActive; } getBackendName() { if (this._nGpuLayers === 0) { return "WASM (Prototype bridge)"; } if (this._backendLabels.length > 0) { return this._backendLabels.join(", "); } return this._gpuActive ? "WebGPU (Prototype bridge)" : "WASM (Prototype bridge)"; } setLogLevel(level) { if (Number.isFinite(level)) { this._logLevel = Math.max(0, Math.min(4, Math.trunc(level))); } this._applyCoreLogLevel(); } cancel() { this._abortRequested = true; if (this._activeTransferAbortController) { try { this._activeTransferAbortController.abort(); } catch (_) { } this._activeTransferAbortController = null; } try { this._core?.ccall("llamadart_webgpu_request_cancel", null, [], []); } catch (_) { } } async dispose() { if (this._textToSpeechActive && this._textToSpeechDone) { this.cancel(); await this._textToSpeechDone; } if (this._activeTransferAbortController) { try { this._activeTransferAbortController.abort(); } catch (_) { } this._activeTransferAbortController = null; } if (this._core) { this._clearPendingMedia(); const mmprojPath = this._mmProjPath; const mmprojFreeRc = Number( this._core.ccall("llamadart_webgpu_mmproj_free", "number", [], []) ); this._core.ccall("llamadart_webgpu_shutdown", null, [], []); this._deleteFsFile(mmprojPath); if (mmprojFreeRc !== 0) { this._emitLogger( "warn", this._coreErrorMessage("Failed to unload multimodal projector during dispose", mmprojFreeRc) ); } } this._modelPath = null; this._modelPaths = []; this._modelBytes = 0; this._modelSource = "network"; this._modelCacheState = "disabled"; this._loadedModelUrl = null; this._mmProjPath = null; this._mmProjSourceUrl = null; this._mmSupportsVision = false; this._mmSupportsAudio = false; this._loraAdaptersLoaded = false; this._abortRequested = false; this._textToSpeechActive = false; this._textToSpeechDone = null; this._resolveTextToSpeechDone = null; this._activeTransferAbortController = null; this._suppressedWarmupWarningCount = 0; this._didReportWarmupWarningSuppression = false; } async applyChatTemplate(messages, addAssistant = true, _customTemplate = null) { return buildPromptFromMessages(messages, addAssistant); } }; // js/src/worker_protocol.ts function serializeWorkerError(error) { if (!error) { return "Unknown worker error"; } if (typeof error === "string") { return error; } if (typeof error.message === "string" && error.message.length > 0) { return error.message; } try { return JSON.stringify(error); } catch (_) { return String(error); } } var bridgeWorkerModeParam = "__llamadartBridgeWorker"; function emptyBridgeState() { return { metadata: {}, contextSize: 0, gpuActive: false, backendName: "WASM (Prototype bridge)", supportsVision: false, supportsAudio: false }; } function snapshotBridgeState(target) { return { metadata: target.getModelMetadata(), contextSize: target.getContextSize(), gpuActive: target.isGpuActive(), backendName: target.getBackendName(), supportsVision: target.supportsVision(), supportsAudio: target.supportsAudio() }; } // js/src/worker_proxy.ts function createBridgeWorkerSource(moduleUrl) { return `import * as workerModule from ${JSON.stringify(moduleUrl)}; if (workerModule && typeof workerModule.enableBridgeWorkerHost === 'function') { workerModule.enableBridgeWorkerHost(); } `; } function resolveWorkerEntryUrl(moduleUrl) { if (typeof moduleUrl !== "string" || moduleUrl.length === 0) { return null; } try { const base = typeof window !== "undefined" && window.location?.href ? window.location.href : void 0; const url = new URL(moduleUrl, base); const path = url.pathname || ""; const usesDedicatedWorkerEntry = path.endsWith("_worker.js"); if (!usesDedicatedWorkerEntry) { url.searchParams.set(bridgeWorkerModeParam, "1"); } return url.toString(); } catch (_) { return null; } } function deriveBridgeModuleUrlFromWorkerEntry(moduleUrl) { if (typeof moduleUrl !== "string" || moduleUrl.length === 0) { return null; } try { const base = typeof window !== "undefined" && window.location?.href ? window.location.href : void 0; const url = new URL(moduleUrl, base); const path = url.pathname || ""; if (!path.endsWith("_worker.js")) { return null; } url.pathname = path.replace(/_worker\.js$/, ".js"); url.searchParams.delete(bridgeWorkerModeParam); return url.toString(); } catch (_) { return null; } } var BridgeWorkerProxy = class { constructor({ moduleUrl, config }) { this._config = config && typeof config === "object" ? config : {}; this._nextId = 1; this._pending = /* @__PURE__ */ new Map(); this._workerBlobUrl = null; let workerInitError = null; const moduleCandidates = [moduleUrl]; const bridgeModuleFallback = deriveBridgeModuleUrlFromWorkerEntry(moduleUrl); if (bridgeModuleFallback && bridgeModuleFallback !== moduleUrl) { moduleCandidates.push(bridgeModuleFallback); } for (const candidate of moduleCandidates) { if (this._worker) { break; } const directWorkerUrl = resolveWorkerEntryUrl(candidate); if (!directWorkerUrl) { continue; } try { this._worker = new Worker(directWorkerUrl, { type: "module" }); } catch (error) { workerInitError = error; } } for (const candidate of moduleCandidates) { if (this._worker) { break; } const source = createBridgeWorkerSource(candidate); this._workerBlobUrl = URL.createObjectURL( new Blob([source], { type: "text/javascript" }) ); try { this._worker = new Worker(this._workerBlobUrl, { type: "module" }); } catch (error) { workerInitError = error; URL.revokeObjectURL(this._workerBlobUrl); this._workerBlobUrl = null; } } if (!this._worker) { throw workerInitError || new Error("Failed to initialize bridge worker"); } this._ready = new Promise((resolve, reject) => { this._readyResolve = resolve; this._readyReject = reject; }); this._readyTimeoutHandle = null; this._armReadyTimeout(); this._worker.onmessage = (event) => { const message = event.data || {}; const type = message.type; if (type === "ready") { this._clearReadyTimeout(); this._readyResolve?.(); return; } const id = Number(message.id || 0); const pending = this._pending.get(id); if (!pending) { return; } if (type === "event") { pending.onEvent?.(message); return; } this._pending.delete(id); if (type === "error") { const errorMessage = String(message.message || "Worker request failed"); const workerError = message.embeddingErrorCode ? new EmbeddingError(message.embeddingErrorCode, errorMessage) : new Error(errorMessage); if (message.state && typeof message.state === "object") { workerError.state = message.state; } pending.reject(workerError); return; } pending.resolve(message); }; this._worker.onerror = (event) => { const message = event?.message || "Bridge worker crashed"; const error = new Error(String(message)); error.llamadartWorkerCrash = true; this._clearReadyTimeout(); this._readyReject?.(error); for (const pending of this._pending.values()) { pending.reject(error); } this._pending.clear(); }; this._worker.postMessage({ type: "init", config }); } async call(method, args, onEvent, transferList = [], operationMeta = null) { await this._ready; const id = this._nextId++; const timeoutMs = this._resolveRequestTimeoutMs(method, args); const transfers = Array.isArray(transferList) ? transferList.filter((item) => item != null) : []; return new Promise((resolve, reject) => { let timeoutHandle = null; const clearTimer = () => { if (timeoutHandle != null) { globalThis.clearTimeout(timeoutHandle); timeoutHandle = null; } }; const armTimer = () => { clearTimer(); if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { return; } timeoutHandle = globalThis.setTimeout(() => { this._pending.delete(id); reject(new Error(`Worker request timeout (${method}, ${timeoutMs}ms)`)); }, timeoutMs); }; armTimer(); this._pending.set(id, { resolve: (value) => { clearTimer(); resolve(value); }, reject: (error) => { clearTimer(); reject(error); }, onEvent: (event) => { armTimer(); onEvent?.(event); } }); const message = { type: "call", id, method, args }; if (operationMeta && typeof operationMeta === "object") { Object.assign(message, operationMeta); } this._worker.postMessage(message, transfers); }); } _resolveWorkerReadyTimeoutMs() { const configured = Number(this._config.workerInitTimeoutMs); if (!Number.isFinite(configured) || configured <= 0) { return 2e4; } return Math.max(3e3, Math.min(12e4, Math.trunc(configured))); } _clearReadyTimeout() { if (this._readyTimeoutHandle != null) { globalThis.clearTimeout(this._readyTimeoutHandle); this._readyTimeoutHandle = null; } } _armReadyTimeout() { this._clearReadyTimeout(); const timeoutMs = this._resolveWorkerReadyTimeoutMs(); this._readyTimeoutHandle = globalThis.setTimeout(() => { this._readyTimeoutHandle = null; const timeoutError = new Error(`Bridge worker init timeout (${timeoutMs}ms)`); this._readyReject?.(timeoutError); try { this._worker?.terminate(); } catch (_) { } }, timeoutMs); } _resolveRequestTimeoutMs(method, args = []) { const explicitGlobal = Number(this._config.workerRequestTimeoutMs); const clamp = (value, fallback) => { if (!Number.isFinite(value) || value <= 0) { return fallback; } return Math.max(5e3, Math.min(36e5, Math.trunc(value))); }; if (method === "loadModelFromUrl" || method === "loadDraftModel") { return clamp(Number(this._config.workerModelLoadTimeoutMs), clamp(explicitGlobal, 3 * 60 * 1e3)); } if (method === "loadMultimodalProjector") { return clamp(Number(this._config.workerMmprojLoadTimeoutMs), clamp(explicitGlobal, 8 * 60 * 1e3)); } if (method === "createCompletion") { return clamp(Number(this._config.workerCompletionTimeoutMs), clamp(explicitGlobal, 6 * 60 * 1e3)); } if (method === "synthesizeSpeech") { return clamp(Number(this._config.workerTextToSpeechTimeoutMs), clamp(explicitGlobal, 20 * 60 * 1e3)); } if (method === "loadDecisionHead" || method === "loadLoraAdapter") { return clamp(explicitGlobal, 10 * 60 * 1e3); } if (method === "runDecision") { const sequences = Array.isArray(args?.[1]) ? args[1].length : 0; return clamp(explicitGlobal, 10 * 60 * 1e3 + sequences * DECISION_WORKER_TIMEOUT_PER_SEQUENCE_MS); } return clamp(explicitGlobal, 12e4); } async dispose() { this._clearReadyTimeout(); let didTimeout = false; try { await Promise.race([ this.call("dispose", []), new Promise((resolve) => { globalThis.setTimeout(() => { didTimeout = true; resolve(null); }, 800); }) ]); } catch (_) { } if (didTimeout) { } for (const pending of this._pending.values()) { pending.reject(new Error("Bridge worker disposed")); } this._pending.clear(); this._worker.terminate(); if (this._workerBlobUrl) { URL.revokeObjectURL(this._workerBlobUrl); this._workerBlobUrl = null; } } }; // js/src/bridge.ts var LlamaWebGpuBridge = class { static supportsSafariAdaptiveGpu = LlamaWebGpuBridgeRuntime.supportsSafariAdaptiveGpu === true; static supportsCompletionUsage = true; constructor(config = {}) { this._config = config; this._runtime = null; this._workerProxy = null; this._workerGeneration = 0; this._workerDisposePromise = null; this._retiringWorkerDisposals = /* @__PURE__ */ new Set(); this._retiredWorkerProxies = /* @__PURE__ */ new WeakSet(); this._workerFallbackReason = null; this._metadata = {}; this._contextSize = 0; this._gpuActive = false; this._backendName = "WASM (Prototype bridge)"; this._supportsVision = false; this._supportsAudio = false; this._loadedModelUrl = null; this._loadedModelOptions = null; this._loadedMmProjUrl = null; this._loadedDraftModel = null; this._multimodalWorkerCpuMode = false; this._workerModelMissing = false; this._bridgeWarnRecent = /* @__PURE__ */ new Map(); this._operationQueueTail = null; this._activeOperation = null; this._nextOperationId = 0; this._lifecycleState = "open"; this._shadowStateTransactionDepth = 0; this._deferredShadowState = null; this._disposed = false; this._disposePromise = null; this._disposalWaiters = /* @__PURE__ */ new Set(); this._decisionHeads = /* @__PURE__ */ new Map(); this._nextDecisionHandle = 1; this._loraAdapters = /* @__PURE__ */ new Map(); this._activeLoraScales = /* @__PURE__ */ new Map(); this._nextLoraHandle = 1; if (this._shouldUseWorker()) { try { this._workerProxy = this._createWorkerProxy(); } catch (error) { this._disableWorkerFallback(error); } } if (this._workerProxy) { this._workerGeneration = 1; } if (!this._workerProxy) { this._runtime = this._createRuntime(); this._captureDirectRuntimeState(); } } _createRuntime() { return new LlamaWebGpuBridgeRuntime({ ...this._config, disableWorker: true }); } _operationSignal(operation = this._activeOperation) { return operation?.abortController?.signal || null; } _recordWorkerGeneration(operation = this._activeOperation, proxy = this._workerProxy) { if (!operation) { return; } const generation = this._workerGeneration || 0; operation.workerGeneration = generation; operation.workerGenerations?.add(generation); if (proxy) { operation.workerProxy = proxy; operation.currentWorkerProxy = proxy; operation.workerProxies?.add(proxy); } } _advanceWorkerGeneration() { this._workerGeneration = (this._workerGeneration || 0) + 1; this._recordWorkerGeneration(); return this._workerGeneration; } _isOperationCurrent(operation) { return !!(operation && this._activeOperation === operation && this._lifecycleState !== "disposed" && operation.state !== "cancelling" && operation.state !== "cancelled" && operation.state !== "disposed"); } _isOperationCallbackCurrent(operation, proxy) { return this._isOperationCurrent(operation) && proxy === this._workerProxy; } _sendWorkerCancel(proxy) { if (!proxy) { return; } if (typeof proxy.call !== "function") { if (proxy === this._workerProxy && typeof this._callWorker === "function") { try { Promise.resolve(this._callWorker("cancel", [])).catch(() => { }); } catch (_) { } } return; } try { Promise.resolve(proxy.call("cancel", [], null)).catch(() => { }); } catch (_) { } } _cancelOperation(operation, reason = "user") { if (!operation) { return; } if (reason === "worker-timeout") { if (operation.cancelRequested || operation.workerTimeoutCancellationSent) { return; } operation.workerTimeoutCancellationSent = true; operation.cancelReason = reason; operation.state = "recovering"; const proxies2 = new Set(operation.workerProxies || []); if (operation.workerProxy) { proxies2.add(operation.workerProxy); } for (const proxy of proxies2) { this._sendWorkerCancel(proxy); } return; } if (operation.cancelRequested) { return; } operation.cancelRequested = true; operation.cancelReason = reason; operation.state = "cancelling"; try { operation.abortController?.abort(); } catch (_) { } const proxies = new Set(operation.workerProxies || []); if (operation.workerProxy) { proxies.add(operation.workerProxy); } for (const proxy of proxies) { this._sendWorkerCancel(proxy); } const runtimes = new Set(operation.runtimes || []); if (operation.runtime) { runtimes.add(operation.runtime); } for (const runtime of runtimes) { try { runtime?.cancel?.(); } catch (_) { } } } _isOperationCancellation(error, operation = this._activeOperation) { if (operation?.callerSignal?.aborted) { return true; } if (operation?.cancelRequested && operation.cancelReason !== "worker-timeout") { return true; } return error?.name === "AbortError" && !this._isWorkerTimeoutError(error); } _throwIfOperationCancelled(error, message = "Bridge operation was cancelled.", operation = this._activeOperation) { if (this._isOperationCancellation(error, operation)) { throw createAbortError(message); } } /** * Guards the success path: a worker that ignored cancellation and settled * with a full result must still reject once the caller's signal was aborted, * and must not commit that result. Only the caller's signal counts here. * A bare `cancel()` stays the documented best-effort control signal, so a * result that still arrived is delivered rather than discarded. */ _throwIfCallerCancelled(message = "Bridge operation was cancelled.", operation = this._activeOperation) { if (operation?.callerSignal?.aborted) { throw createAbortError(message); } } _retireWorkerProxy(proxy) { if (!proxy || typeof proxy !== "object") { return Promise.resolve(); } if (this._retiredWorkerProxies?.has(proxy)) { return Promise.resolve(); } this._retiredWorkerProxies?.add(proxy); const operation = this._activeOperation; operation?.workerProxies?.add(proxy); let disposal; try { disposal = Promise.resolve(proxy.dispose?.()).catch(() => { }); } catch (_) { disposal = Promise.resolve(); } this._retiringWorkerDisposals ??= /* @__PURE__ */ new Set(); this._retiringWorkerDisposals.add(disposal); this._workerDisposePromise = disposal; operation?.retirements?.add(disposal); disposal.then( () => { this._retiringWorkerDisposals.delete(disposal); if (this._workerDisposePromise === disposal && this._retiringWorkerDisposals.size === 0) { this._workerDisposePromise = null; } }, () => { this._retiringWorkerDisposals.delete(disposal); if (this._workerDisposePromise === disposal && this._retiringWorkerDisposals.size === 0) { this._workerDisposePromise = null; } } ); return disposal; } async _waitForWorkerDisposal() { this._retiringWorkerDisposals ??= /* @__PURE__ */ new Set(); while (this._retiringWorkerDisposals.size > 0) { const pending = [...this._retiringWorkerDisposals]; await Promise.all(pending); } this._workerDisposePromise = null; } /** * llama.cpp exposes one model/context per bridge instance and publishes * results through process-global C buffers (`last_output`, `last_tokens_json`, * `last_detokenized`, `last_embedding_json`, ...). Every async call that reads * or mutates that runtime therefore has to own it exclusively for its whole * duration: a second caller interleaving between the work ccall and the buffer * read would otherwise observe the other caller's result. This FIFO queue is * the single-writer lock, shared by the worker and direct-runtime paths. * * A caller whose signal is already aborted, or that aborts while still queued, * never runs and never dispatches; its slot is skipped in turn without * touching the unrelated operation that currently owns the runtime. Once an * operation has started, its signal is routed to `_cancelOperation` so the * cancel reaches the worker generation and runtime that operation captured * rather than whatever topology happens to be current. */ _runExclusive(run, options = {}) { const signal = options.signal || null; const abortMessage = options.abortMessage || "Bridge operation was cancelled."; const allowDisposed = options.allowDisposed === true; const abortError = () => createAbortError(abortMessage); const lifecycleState = this._lifecycleState || (this._disposed ? "disposed" : "open"); if (!allowDisposed && (this._disposed || lifecycleState !== "open")) { return Promise.reject(new Error(BRIDGE_DISPOSED_MESSAGE)); } if (signal?.aborted) { return Promise.reject(abortError()); } const operation = { id: ++this._nextOperationId, kind: options.kind || "bridge-operation", state: "queued", callerSignal: signal, abortController: typeof AbortController === "function" ? new AbortController() : null, cancelRequested: false, cancelReason: null, workerTimeoutCancellationSent: false, workerGeneration: this._workerGeneration || 0, workerGenerations: /* @__PURE__ */ new Set([this._workerGeneration || 0]), workerProxy: this._workerProxy || null, runtime: this._runtime || null, workerProxies: new Set(this._workerProxy ? [this._workerProxy] : []), runtimes: new Set(this._runtime ? [this._runtime] : []), modelSource: null, modelOptions: null, projectorSource: null, recoveryAttempted: false, generationStarted: false, retirements: /* @__PURE__ */ new Set(), deferredShadowState: null }; const predecessor = this._operationQueueTail; let releaseSuccessor = () => { }; const queueSlot = new Promise((resolve) => { releaseSuccessor = () => resolve(void 0); }); this._operationQueueTail = queueSlot; let started = false; let removeAbortListener = null; let removeDisposalWaiter = null; const detachWatchers = () => { removeAbortListener?.(); removeDisposalWaiter?.(); removeAbortListener = null; removeDisposalWaiter = null; }; const watchesAbort = signal != null && typeof signal.addEventListener === "function"; const watchesDisposal = !allowDisposed; const ordered = (async () => { if (predecessor) { await predecessor; } if (signal?.aborted) { operation.state = "skipped"; throw abortError(); } if (!allowDisposed && (this._disposed || this._lifecycleState === "disposing")) { operation.state = "disposed"; throw new Error(BRIDGE_DISPOSED_MESSAGE); } started = true; removeDisposalWaiter?.(); removeDisposalWaiter = null; operation.state = "starting"; operation.workerProxy = this._workerProxy || null; operation.workerGeneration = this._workerGeneration || 0; operation.workerGenerations.add(operation.workerGeneration); operation.runtime = this._runtime || null; if (operation.workerProxy) { operation.workerProxies.add(operation.workerProxy); operation.currentWorkerProxy = operation.workerProxy; } if (operation.runtime) { operation.runtimes.add(operation.runtime); } this._activeOperation = operation; operation.state = "running"; let operationFailed = false; try { return await run(operation); } catch (error) { operationFailed = true; throw error; } finally { let terminalState = operation.cancelRequested ? "cancelled" : operationFailed ? "failed" : "completed"; operation.state = terminalState === "completed" ? "settling" : terminalState; try { this._captureDirectRuntimeState(); } catch (error) { terminalState = "failed"; throw error; } finally { if (this._activeOperation === operation) { this._activeOperation = null; } operation.state = terminalState; } } })(); const release = () => { if (this._operationQueueTail === queueSlot) { this._operationQueueTail = null; } releaseSuccessor(); }; ordered.then(release, release); if (!watchesAbort && !watchesDisposal) { return ordered; } const abandonedWhileQueued = new Promise((_, reject) => { if (watchesAbort) { const onAbort = () => { if (!started) { operation.state = "skipped"; reject(abortError()); return; } this._cancelOperation(operation, "signal"); }; signal.addEventListener("abort", onAbort, { once: true }); removeAbortListener = () => signal.removeEventListener("abort", onAbort); if (signal.aborted) { onAbort(); } } if (watchesDisposal && !started) { removeDisposalWaiter = this._addDisposalWaiter(() => { if (!started) { operation.state = "disposed"; reject(new Error(BRIDGE_DISPOSED_MESSAGE)); } }); } }); const raced = Promise.race([ordered, abandonedWhileQueued]); raced.then(detachWatchers, detachWatchers); return raced; } /** * Registers a disposal listener for the lifetime of one pending operation and * returns its remover. A listener set is used rather than a long-lived promise * because a promise reaction cannot be detached. */ _addDisposalWaiter(onDisposed) { const waiters = this._disposalWaiters; waiters.add(onDisposed); return () => { waiters.delete(onDisposed); }; } /** * True while `operation` is the non-dispose owner of the queue slot. Teardown * is queued behind it, so it is allowed to finish the work it already started * even once disposal has been requested. */ _isSettleFirstOwner(operation = this._activeOperation) { return !!(operation && this._activeOperation === operation && operation.kind !== "dispose" && operation.state !== "cancelled" && operation.state !== "disposed"); } /** * Lifecycle gate for the helpers that stay off the queue. They must not * dispatch to a torn-down worker, and must not lazily recreate the runtime * disposal just released. `disposing` still admits the operation that already * owns the slot, because disposal settles it first rather than interrupting * it; once teardown has run, every caller is rejected. */ _throwIfDisposed() { if (this._lifecycleState === "disposed") { throw new Error(BRIDGE_DISPOSED_MESSAGE); } if ((this._disposed || this._lifecycleState === "disposing") && !this._isSettleFirstOwner()) { throw new Error(BRIDGE_DISPOSED_MESSAGE); } } _notifyDisposalWaiters() { const waiters = this._disposalWaiters; if (waiters.size === 0) { return; } this._disposalWaiters = /* @__PURE__ */ new Set(); for (const waiter of waiters) { waiter(); } } /** * Mirror the direct runtime's model/context state into facade fields so the * synchronous getters can answer from cache. Reading them straight from the * runtime would ccall into the singleton core outside the queue, racing * whichever async operation currently owns it. */ _captureDirectRuntimeState() { if (this._workerProxy || !this._runtime) { return; } try { this._metadata = this._runtime.getModelMetadata(); this._contextSize = this._runtime.getContextSize(); this._gpuActive = this._runtime.isGpuActive(); this._backendName = this._runtime.getBackendName(); this._supportsVision = this._runtime.supportsVision(); this._supportsAudio = this._runtime.supportsAudio(); } catch (_) { } } /** * prefetch/evict stay off the queue because they only touch Cache Storage and * the network. In direct mode their plain-JS cache bookkeeping is copied into * the facade snapshot. A worker-backed snapshot remains wholly worker-owned so * a separate cache-only runtime cannot misdescribe the active worker model. */ _refreshCacheMetadataSnapshot() { const runtime = this._runtime; if (this._workerProxy || !runtime) { return; } this._metadata = { ...this._metadata || {}, "llamadart.webgpu.model_source": String(runtime._modelSource ?? ""), "llamadart.webgpu.model_cache_state": String(runtime._modelCacheState ?? ""), "llamadart.webgpu.model_cache_name": String(runtime._modelCacheName ?? ""), "llamadart.webgpu.runtime_notes": Array.isArray(runtime._runtimeNotes) ? runtime._runtimeNotes.join(";") : "" }; } _sanitizeModelLoadOptions(options = {}) { const source = options && typeof options === "object" ? options : {}; const sanitized = { ...source }; delete sanitized.progressCallback; delete sanitized.signal; return sanitized; } _createCpuSafeMultimodalLoadOptions(options = {}) { const sanitized = this._sanitizeModelLoadOptions(options); sanitized.nGpuLayers = 0; if (Number.isFinite(Number(sanitized.nCtx)) && Number(sanitized.nCtx) > 4096) { sanitized.nCtx = 4096; } if (!Number.isFinite(Number(sanitized.nThreads)) || Number(sanitized.nThreads) <= 0) { sanitized.nThreads = 4; } else { sanitized.nThreads = Math.min(4, Math.max(1, Math.trunc(Number(sanitized.nThreads)))); } sanitized.nThreadsBatch = sanitized.nThreads; if (!Number.isFinite(Number(sanitized.nBatch)) || Number(sanitized.nBatch) <= 0) { sanitized.nBatch = 128; } else { sanitized.nBatch = Math.min(128, Math.max(32, Math.trunc(Number(sanitized.nBatch)))); } if (!Number.isFinite(Number(sanitized.nUbatch)) || Number(sanitized.nUbatch) <= 0) { sanitized.nUbatch = Math.min(64, sanitized.nBatch); } else { sanitized.nUbatch = Math.min( sanitized.nBatch, Math.min(64, Math.max(1, Math.trunc(Number(sanitized.nUbatch)))) ); } return sanitized; } _rememberLoadedModel(url, options = {}) { const normalizedUrl = cloneModelSource(url); if (!hasModelSource(normalizedUrl)) { return; } this._loadedModelUrl = normalizedUrl; this._loadedModelOptions = this._sanitizeModelLoadOptions(options); this._loadedMmProjUrl = null; this._forgetLoraAdapters(); this._loadedDraftModel = null; this._multimodalWorkerCpuMode = this._workerProxy != null; this._workerModelMissing = false; if (this._activeOperation) { this._activeOperation.modelSource = cloneModelSource(normalizedUrl); this._activeOperation.modelOptions = { ...this._loadedModelOptions }; this._activeOperation.projectorSource = null; } } _forgetLoadedModel() { this._loadedModelUrl = null; this._loadedModelOptions = null; this._loadedMmProjUrl = null; this._forgetLoraAdapters(); this._loadedDraftModel = null; this._multimodalWorkerCpuMode = false; this._workerModelMissing = false; } /** * A failed load can leave its target without any model: the runtime drops * the previous model once the new download starts. Keep the facade's model * source only while the target still reports a loaded model, so recovery * never reloads a model the target no longer holds. * * A worker's reply is read from `attempt`, not the facade snapshot: a * cancelled operation no longer accepts worker state, so the snapshot can * still describe a model the worker already dropped. Only the bookkeeping is * corrected here; a cancelled load never publishes the worker's snapshot. */ _syncLoadedModelAfterFailedLoad(attempt) { const proxy = this._workerProxy; if (!proxy) { if (!(Number(this._runtime?._modelBytes) > 0)) { this._forgetLoadedModel(); } return; } const fromCurrentWorker = attempt.proxy === proxy; const replacedModel = fromCurrentWorker && attempt.loaded; const state = fromCurrentWorker ? attempt.state : null; const modelBytes = Number( (state ? state.metadata : this._metadata)?.["llamadart.webgpu.model_bytes"] ); if (replacedModel || !(modelBytes > 0)) { this._forgetLoadedModel(); } } _rememberLoadedMmProj(url) { const normalizedUrl = String(url || "").trim(); if (normalizedUrl.length === 0) { return; } this._loadedMmProjUrl = normalizedUrl; if (this._activeOperation) { this._activeOperation.projectorSource = normalizedUrl; } } _hasMediaParts(options = {}) { return Array.isArray(options?.parts) && options.parts.length > 0; } async _replaceWorkerProxyForMultimodalCpuMode() { this._throwIfDisposed(); const hadWorkerProxy = this._workerProxy != null; if (hadWorkerProxy) { const staleProxy = this._workerProxy; this._advanceWorkerGeneration(); this._workerProxy = null; this._retireWorkerProxy(staleProxy); await this._waitForWorkerDisposal(); this._throwIfDisposed(); } let replacement; try { replacement = this._createWorkerProxy(); } catch (error) { this._disableWorkerFallback(error); throw error; } try { this._throwIfDisposed(); } catch (error) { this._retireWorkerProxy(replacement); throw error; } if (!hadWorkerProxy) { this._advanceWorkerGeneration(); } this._workerProxy = replacement; this._recordWorkerGeneration(this._activeOperation, replacement); this._multimodalWorkerCpuMode = false; this._workerModelMissing = true; } _isRecoverableWorkerFsError(error) { const text = serializeWorkerError(error).toLowerCase(); if (text.includes("model shard:")) { return false; } return text.includes("fs error") || text.includes("no such file") || text.includes("not found") || text.includes("invalid argument") || text.includes("timed out"); } _isWorkerRequestTimeoutError(error) { const text = serializeWorkerError(error).toLowerCase(); return text.includes("worker request timeout") || text.includes("worker init timeout") || text.includes("worker timed out"); } async _ensureWorkerMultimodalCpuMode() { this._throwIfDisposed(); if (!this._workerProxy) { await this._replaceWorkerProxyForMultimodalCpuMode(); } this._throwIfDisposed(); if (this._multimodalWorkerCpuMode) { return true; } if (!hasModelSource(this._loadedModelUrl)) { return false; } const selectedOptions = this._sanitizeModelLoadOptions( this._loadedModelOptions || {} ); const operation = this._activeOperation; if (operation) { operation.recoveryAttempted = true; operation.state = "recovering"; } const previousDeferredState = this._deferredShadowState; this._shadowStateTransactionDepth += 1; this._deferredShadowState = null; let recoverySucceeded = false; const applyWorkerSafeMode = () => this._loadRememberedModelIntoWorker(selectedOptions); try { await applyWorkerSafeMode(); recoverySucceeded = true; this._emitBridgeWarn( "llamadart: multimodal worker prepared in selected backend mode." ); return true; } catch (error) { this._throwIfDisposed(); this._emitBridgeWarn( `llamadart: multimodal worker setup failed once; restarting worker (${serializeWorkerError(error)}).` ); await this._replaceWorkerProxyForMultimodalCpuMode(); await applyWorkerSafeMode(); recoverySucceeded = true; this._emitBridgeWarn( "llamadart: multimodal worker recovered after restart." ); return true; } finally { this._shadowStateTransactionDepth = Math.max(0, this._shadowStateTransactionDepth - 1); if (this._shadowStateTransactionDepth === 0) { if (recoverySucceeded && this._deferredShadowState && this._workerProxy) { this._applyShadowStateFrom(this._workerProxy, this._deferredShadowState); } this._deferredShadowState = previousDeferredState || null; } if (operation && this._activeOperation === operation && operation.state === "recovering") { operation.state = operation.cancelRequested ? "cancelling" : "running"; } } } async _loadRememberedModelIntoWorker(selectedOptions) { this._throwIfDisposed(); this._dropLoraAdaptersOf(this._workerProxy); await this._callWorker("loadModelFromUrl", [this._loadedModelUrl, selectedOptions]); this._throwIfDisposed(); if (typeof this._loadedMmProjUrl === "string" && this._loadedMmProjUrl.length > 0) { await this._callWorker("loadMultimodalProjector", [this._loadedMmProjUrl]); } const draft = this._loadedDraftModel; if (draft) { try { await this._callWorker("loadDraftModel", [draft.url, draft.options]); } catch (error) { this._throwIfOperationCancelled(error, "Draft model reload was cancelled."); this._forgetDraftModelAfterReloadFailure(error); } } this._throwIfDisposed(); await this._restoreLoraAdapters(); this._throwIfDisposed(); this._workerModelMissing = false; this._loadedModelOptions = selectedOptions; this._multimodalWorkerCpuMode = true; } /** * Multimodal recovery can replace the worker and then fail to reload the * model, its projector or its applied LoRA adapters into it. The facade * still holds that model, but the fresh worker would answer every request * with "No model loaded" or without the adapters. Reload them first, and * again on the next call if this fails. A failure here reaches the caller's * own worker-error handling. */ async _restoreWorkerModelIfMissing() { if (!this._workerProxy || this._workerModelMissing !== true || !hasModelSource(this._loadedModelUrl)) { return; } this._emitBridgeWarn( "llamadart: bridge worker was restarted without its model; reloading it." ); await this._loadRememberedModelIntoWorker( this._sanitizeModelLoadOptions(this._loadedModelOptions || {}) ); } _isDispatchWorkgroupLimitError(error) { const text = serializeWorkerError(error).toLowerCase(); return text.includes("dispatch workgroup count") || text.includes("max compute workgroups per dimension") || text.includes("invalid commandbuffer") || text.includes("ggml_webgpu: device error") || text.includes("runtimeerror: aborted") || text.includes("aborted()"); } // An aborted or trapped Wasm core stays dead. The worker host posts only // the message text, so a worker-side RuntimeError arrives without its name. _isWasmCoreAbortError(error) { if (typeof WebAssembly !== "undefined" && error instanceof WebAssembly.RuntimeError) { return true; } const text = serializeWorkerError(error).toLowerCase(); return text.includes("aborted(") || text.includes("runtimeerror") || text.includes("unreachable") || text.includes("memory access out of bounds") || text.includes("function signature mismatch") || text.includes("program terminated with exit"); } // A worker request falls back only when the worker can no longer serve // requests: it crashed, stalled, timed out, never initialized, or its core // aborted. A core error from a healthy worker (invalid grammar, a context // limit, an unsupported template, a model without embeddings) is // deterministic; retrying on the main thread fails the same way and strands // the session there. _isWorkerUnusableError(error) { if (error && typeof error === "object" && error.llamadartWorkerCrash === true) { return true; } return this._isWorkerTimeoutError(error) || this._isWasmCoreAbortError(error) || this._shouldFallbackToMainThread(error); } _isWorkerTimeoutError(error) { if (error && typeof error === "object" && error.llamadartWorkerTimeout === true) { return true; } const text = serializeWorkerError(error).toLowerCase(); return text.includes("worker completion stalled") || text.includes("worker createcompletion stalled") || text.includes("worker timed out") || text.includes("worker timeout"); } _isCpuModelMode() { const requestedLayers = Number(this._loadedModelOptions?.nGpuLayers); if (Number.isFinite(requestedLayers)) { return requestedLayers === 0; } const metadataLayers = Number(this._metadata?.["llamadart.webgpu.n_gpu_layers"]); if (Number.isFinite(metadataLayers)) { return metadataLayers === 0; } return false; } _workerCompletionStallTimeoutMs(options = {}) { const configured = Number(this._config?.workerGenerationStallTimeoutMs); if (Number.isFinite(configured) && configured > 0) { return Math.max(5e3, Math.min(3e5, Math.trunc(configured))); } if (!this._hasMediaParts(options)) { return 9e4; } if (this._isCpuModelMode()) { return 0; } return 18e4; } async _ensureRuntimeReadyAfterWorkerFallback(options = {}, fallbackError = null) { this._throwIfDisposed(); await this._waitForWorkerDisposal(); this._throwIfDisposed(); if (!this._runtime) { this._runtime = this._createRuntime(); } const operation = this._activeOperation; operation?.runtimes?.add(this._runtime); if (operation && operation.state === "running") { operation.state = "recovering"; } const forceReloadRequested = options?._llamadartForceRuntimeReload === true; const mediaPartsRequested = this._hasMediaParts(options); const textToSpeechRequested = options?._llamadartTextToSpeech === true; const multimodalRuntimeRequired = mediaPartsRequested || textToSpeechRequested; const shouldEnsureMultimodalInRuntime = multimodalRuntimeRequired && typeof this._loadedMmProjUrl === "string" && this._loadedMmProjUrl.length > 0; const workerTimedOut = this._isWorkerTimeoutError(fallbackError); const dispatchWorkgroupFallback = this._isDispatchWorkgroupLimitError(fallbackError); const loadedGpuLayers = Number(this._loadedModelOptions?.nGpuLayers); const metadataGpuLayers = Number(this._metadata?.["llamadart.webgpu.n_gpu_layers"]); const modelLoadedWithGpu = Number.isFinite(loadedGpuLayers) ? loadedGpuLayers !== 0 : Number.isFinite(metadataGpuLayers) ? metadataGpuLayers !== 0 : true; const shouldUseCpuMultimodalFallback = multimodalRuntimeRequired && modelLoadedWithGpu && (dispatchWorkgroupFallback || workerTimedOut); try { if (Number(this._runtime?._modelBytes) > 0 && !forceReloadRequested && !shouldUseCpuMultimodalFallback) { await this._ensureRuntimeDraftModel(); if (shouldEnsureMultimodalInRuntime) { const runtimeSupportsMedia = typeof this._runtime.supportsVision === "function" && this._runtime.supportsVision() || typeof this._runtime.supportsAudio === "function" && this._runtime.supportsAudio(); if (!runtimeSupportsMedia) { this._throwIfDisposed(); await this._runtime.loadMultimodalProjector(this._loadedMmProjUrl); this._throwIfDisposed(); if (Array.isArray(this._runtime._runtimeNotes)) { this._runtime._runtimeNotes.push("worker_fallback_reload_mmproj"); } } } await this._restoreLoraAdapters(); this._throwIfDisposed(); return; } if (!hasModelSource(this._loadedModelUrl)) { return; } const loadOptions = shouldUseCpuMultimodalFallback ? this._createCpuSafeMultimodalLoadOptions(this._loadedModelOptions || {}) : this._sanitizeModelLoadOptions(this._loadedModelOptions || {}); const recoverySignal = this._operationSignal(operation) || options.signal || null; if (recoverySignal) { loadOptions.signal = recoverySignal; } if (shouldUseCpuMultimodalFallback) { if (textToSpeechRequested) { this._emitBridgeWarn( "llamadart: retrying text-to-speech once with CPU fallback after WebGPU failure." ); } else if (workerTimedOut) { this._emitBridgeWarn( "llamadart: retrying multimodal generation with CPU fallback after worker timeout." ); } else { this._emitBridgeWarn( "llamadart: retrying multimodal generation with CPU fallback after WebGPU workgroup limit failure." ); } } if (workerTimedOut) { this._emitBridgeWarn( "llamadart: bridge worker completion stalled; restarting generation path on main-thread runtime." ); } this._throwIfDisposed(); this._dropLoraAdaptersOf(this._runtime); await this._runtime.loadModelFromUrl(this._loadedModelUrl, loadOptions); this._throwIfDisposed(); if (Array.isArray(this._runtime._runtimeNotes)) { this._runtime._runtimeNotes.push("worker_fallback_reload_model"); if (forceReloadRequested) { this._runtime._runtimeNotes.push("worker_fallback_reload_forced"); } if (workerTimedOut) { this._runtime._runtimeNotes.push("worker_fallback_timeout"); } if (shouldUseCpuMultimodalFallback) { this._runtime._runtimeNotes.push("worker_fallback_cpu_multimodal"); if (textToSpeechRequested) { this._runtime._runtimeNotes.push("worker_fallback_cpu_text_to_speech"); } } } if (shouldEnsureMultimodalInRuntime) { this._throwIfDisposed(); await this._runtime.loadMultimodalProjector(this._loadedMmProjUrl); this._throwIfDisposed(); } await this._restoreLoraAdapters(); this._throwIfDisposed(); await this._ensureRuntimeDraftModel(); } catch (error) { if (this._disposed || this._lifecycleState === "disposing") { throw new Error(BRIDGE_DISPOSED_MESSAGE); } if (error?.name === "AbortError") { throw error; } try { await this._runtime?.dispose?.(); } catch (_) { } this._runtime = null; this._metadata = {}; this._contextSize = 0; this._gpuActive = false; this._backendName = ""; this._supportsVision = false; this._supportsAudio = false; this._loadedModelUrl = null; this._loadedModelOptions = null; this._loadedMmProjUrl = null; this._forgetLoraAdapters(); this._loadedDraftModel = null; if (!this._disposed && this._lifecycleState === "open" && !this._workerProxy) { this._runtime = this._createRuntime(); operation?.runtimes?.add(this._runtime); } throw error; } finally { if (operation && this._activeOperation === operation && operation.state === "recovering") { operation.state = operation.cancelRequested ? "cancelling" : "running"; } } } _createWorkerProxy() { return new BridgeWorkerProxy({ moduleUrl: this._workerModuleUrl(), config: this._workerConfig() }); } _shouldUseWorker() { if (this._config?.disableWorker === true) { return false; } if (typeof Worker === "undefined" || typeof Blob === "undefined" || typeof URL === "undefined" || typeof URL.createObjectURL !== "function") { return false; } if (this._config?.coreModuleFactory != null) { return false; } return true; } _workerModuleUrl() { const candidate = this._config?.workerUrl; if (typeof candidate === "string" && candidate.trim().length > 0) { return candidate.trim(); } try { return new URL("./llama_webgpu_bridge_worker.js", import.meta.url).toString(); } catch (_) { return import.meta.url; } } _workerConfig() { const config = this._config || {}; return { workerRequestTimeoutMs: config.workerRequestTimeoutMs, workerInitTimeoutMs: config.workerInitTimeoutMs, workerModelLoadTimeoutMs: config.workerModelLoadTimeoutMs, workerMmprojLoadTimeoutMs: config.workerMmprojLoadTimeoutMs, workerCompletionTimeoutMs: config.workerCompletionTimeoutMs, workerTextToSpeechTimeoutMs: config.workerTextToSpeechTimeoutMs, coreInitTimeoutMs: config.coreInitTimeoutMs, wasmUrl: typeof config.wasmUrl === "string" ? config.wasmUrl : void 0, wasmUrlMem64: typeof config.wasmUrlMem64 === "string" ? config.wasmUrlMem64 : void 0, coreModuleUrl: typeof config.coreModuleUrl === "string" ? config.coreModuleUrl : void 0, coreModuleUrlMem64: typeof config.coreModuleUrlMem64 === "string" ? config.coreModuleUrlMem64 : void 0, preferMemory64: typeof config.preferMemory64 === "boolean" ? config.preferMemory64 : void 0, threads: Number(config.threads) > 0 ? Number(config.threads) : void 0, threadsBatch: Number(config.threadsBatch) > 0 ? Number(config.threadsBatch) : void 0, threadPoolSize: Number(config.threadPoolSize) > 0 ? Number(config.threadPoolSize) : void 0, nBatch: Number(config.nBatch) > 0 ? Number(config.nBatch) : void 0, nUbatch: Number(config.nUbatch) > 0 ? Number(config.nUbatch) : void 0, nGpuLayers: Number.isFinite(config.nGpuLayers) ? Number(config.nGpuLayers) : void 0, userAgent: typeof config.userAgent === "string" ? config.userAgent : void 0, cacheName: typeof config.cacheName === "string" ? config.cacheName : void 0, remoteFetchThresholdBytes: Number(config.remoteFetchThresholdBytes) > 0 ? Number(config.remoteFetchThresholdBytes) : void 0, remoteFetchChunkBytes: Number(config.remoteFetchChunkBytes) > 0 ? Number(config.remoteFetchChunkBytes) : void 0, mediaMaxImagePixels: Number(config.mediaMaxImagePixels) > 0 ? Number(config.mediaMaxImagePixels) : void 0, mediaMaxImageEdge: Number(config.mediaMaxImageEdge) > 0 ? Number(config.mediaMaxImageEdge) : void 0, disableImageDownscale: config.disableImageDownscale === true, allowAutoRemoteFetchBackend: config.allowAutoRemoteFetchBackend === true, logLevel: Number.isFinite(config.logLevel) ? Number(config.logLevel) : 2 }; } _applyShadowState(state) { if (!state || typeof state !== "object") { return; } if (state.metadata && typeof state.metadata === "object") { this._metadata = state.metadata; } if (Number.isFinite(state.contextSize)) { this._contextSize = Number(state.contextSize); } if (typeof state.gpuActive === "boolean") { this._gpuActive = state.gpuActive; } if (typeof state.backendName === "string" && state.backendName.length > 0) { this._backendName = state.backendName; } if (typeof state.supportsVision === "boolean") { this._supportsVision = state.supportsVision; } if (typeof state.supportsAudio === "boolean") { this._supportsAudio = state.supportsAudio; } } _shouldFallbackToMainThread(error) { const text = serializeWorkerError(error).toLowerCase(); if (text.includes("aborted(native code called abort())")) { return false; } if (text.includes("array buffer allocation failed")) { return false; } if (text.includes("bad_alloc")) { return false; } if (text.includes("out of memory")) { return false; } if (text.includes("memory access out of bounds")) { return false; } if (text.includes("bridge worker")) { return true; } if (text.includes("worker request failed")) { return true; } if (text.includes("worker request timeout")) { return true; } if (text.includes("worker init timeout")) { return true; } if (text.includes("timed out")) { return true; } if (text.includes("worker proxy is not available")) { return true; } if (text.includes("worker is not initialized")) { return true; } if (text.includes("failed to initialize bridge worker")) { return true; } if (text.includes("script error")) { return true; } return false; } _resolvedBridgeLogLevel() { const configured = Number(this._config?.logLevel); if (Number.isFinite(configured)) { return Math.max(0, Math.min(4, Math.trunc(configured))); } const runtimeLevel = Number(this._runtime?._logLevel); if (Number.isFinite(runtimeLevel)) { return Math.max(0, Math.min(4, Math.trunc(runtimeLevel))); } return 2; } _shouldEmitBridgeLevel(level) { const configured = this._resolvedBridgeLogLevel(); const threshold = logThresholdForConfiguredLevel(configured); if (threshold > 3) { return false; } return logLevelForName(level) >= threshold; } _shouldSuppressBridgeWarn(message) { const text = String(message || "").trim(); if (text.length === 0) { return false; } const configuredWindow = Number(this._config?.warnDedupWindowMs); const dedupWindowMs = Number.isFinite(configuredWindow) && configuredWindow > 0 ? Math.max(500, Math.min(6e4, Math.trunc(configuredWindow))) : 5e3; const now = Date.now(); const last = Number(this._bridgeWarnRecent.get(text) || 0); this._bridgeWarnRecent.set(text, now); if (this._bridgeWarnRecent.size > 80) { const staleThreshold = now - dedupWindowMs * 2; for (const [key, atMs] of this._bridgeWarnRecent.entries()) { if (Number(atMs) < staleThreshold) { this._bridgeWarnRecent.delete(key); } } } return last > 0 && now - last < dedupWindowMs; } _emitBridgeWarn(message) { if (!this._shouldEmitBridgeLevel("warn")) { return; } if (this._shouldSuppressBridgeWarn(message)) { return; } if (this._runtime && typeof this._runtime._emitLogger === "function") { this._runtime._emitLogger("warn", message); return; } if (typeof console !== "undefined" && typeof console.warn === "function") { console.warn(message); } } _disableWorkerFallback(error) { const reason = serializeWorkerError(error); this._workerFallbackReason = reason; if (typeof globalThis !== "undefined") { globalThis.__llamadartBridgeWorkerFallbackReason = reason; } this._emitBridgeWarn( `llamadart: bridge worker unavailable, falling back to main thread (${reason})` ); if (this._workerProxy) { const workerProxy = this._workerProxy; this._advanceWorkerGeneration(); this._workerProxy = null; this._retireWorkerProxy(workerProxy); } this._multimodalWorkerCpuMode = false; this._workerModelMissing = false; const activeOperationMayFinish = this._activeOperation && this._activeOperation.kind !== "dispose" && this._activeOperation.state !== "cancelled" && this._activeOperation.state !== "disposed"; if ((this._disposed || this._lifecycleState === "disposing") && !activeOperationMayFinish) { return; } if (!this._runtime) { this._runtime = this._createRuntime(); this._activeOperation?.runtimes?.add(this._runtime); } const note = `worker_fallback:${reason}`; if (this._runtime && Array.isArray(this._runtime._runtimeNotes) && typeof reason === "string" && reason.length > 0 && !this._runtime._runtimeNotes.includes(note)) { this._runtime._runtimeNotes.push(note); } } async _callWorker(method, args, onEvent, transferList = []) { const proxy = this._workerProxy; if (!proxy) { throw new Error("Bridge worker proxy is not available"); } const operation = this._activeOperation; if (operation) { this._recordWorkerGeneration(operation, proxy); } const guardedEvent = typeof onEvent === "function" ? (event) => { if (!operation || this._isOperationCallbackCurrent(operation, proxy) || event.event === "usage" && proxy === this._workerProxy) { onEvent(event); } } : void 0; try { const response = await proxy.call( method, args, guardedEvent, transferList, operation ? { operationId: operation.id, workerGeneration: operation.workerGeneration } : void 0 ); if (response?.state) { this._acceptWorkerState(operation, proxy, response.state); } return response?.value; } catch (error) { if (error && typeof error === "object" && error.state) { this._acceptWorkerState(operation, proxy, error.state); } throw error; } } /** * A worker response may only reach facade state while its proxy is still the * operation's current owner. Staging it for an open recovery transaction is * the same commit deferred, so a retired generation must be rejected before * the transaction branch, not after it. */ _acceptWorkerState(operation, proxy, state) { if (operation && !this._isOperationCallbackCurrent(operation, proxy)) { return; } if (this._shadowStateTransactionDepth > 0) { this._deferredShadowState = state; if (operation) { operation.deferredShadowState = state; } return; } this._applyShadowStateFrom(proxy, state); } /** * The getters now answer from the facade snapshot, so a response from a proxy * that has since been replaced must not write into the current worker's state. */ _applyShadowStateFrom(proxy, state) { if (proxy !== this._workerProxy) { return; } this._applyShadowState(state); } async loadModelFromUrl(url, options = {}) { return this._runExclusive( () => this._loadModelFromUrlUnlocked(url, options), { signal: options?.signal, abortMessage: "Model load was cancelled.", kind: "load-model" } ); } async _loadModelFromUrlUnlocked(url, options = {}) { const attempt = { proxy: null, state: null, loaded: false }; try { return await this._loadModelFromUrlOnTarget(url, options, attempt); } catch (error) { this._syncLoadedModelAfterFailedLoad(attempt); throw error; } } /** * @param attempt Records the last worker load's proxy and reply for * `_syncLoadedModelAfterFailedLoad`. */ async _loadModelFromUrlOnTarget(url, options, attempt) { if (!this._workerProxy) { const runtimeOptions = { ...options, signal: this._operationSignal() || options.signal || null }; const result = await this._runtime.loadModelFromUrl(url, runtimeOptions); this._rememberLoadedModel(url, options); return result; } const invokeWorkerLoad = async () => { const workerOptions = { ...options }; delete workerOptions.progressCallback; delete workerOptions.signal; const proxy = this._workerProxy; attempt.proxy = proxy; attempt.state = null; attempt.loaded = false; let result; try { result = await this._callWorker( "loadModelFromUrl", [url, workerOptions], (event) => { if (event.event !== "progress") { return; } if (typeof options.progressCallback !== "function") { return; } options.progressCallback(event.payload || {}); } ); } catch (error) { attempt.state = error && typeof error === "object" ? error.state || null : null; throw error; } attempt.loaded = true; this._throwIfCallerCancelled("Model load was cancelled."); this._rememberLoadedModel(url, workerOptions); return result; }; try { return await invokeWorkerLoad(); } catch (error) { this._throwIfOperationCancelled(error, "Model load was cancelled."); if (this._isRecoverableWorkerFsError(error) && !this._isWorkerRequestTimeoutError(error)) { this._emitBridgeWarn( `llamadart: worker model-load FS error detected; restarting worker (${serializeWorkerError(error)}).` ); try { await this._replaceWorkerProxyForMultimodalCpuMode(); this._forgetLoadedModel(); this._applyShadowState(emptyBridgeState()); return await invokeWorkerLoad(); } catch (retryError) { this._emitBridgeWarn( `llamadart: worker model-load retry failed (${serializeWorkerError(retryError)}).` ); error = retryError; } } if (!this._shouldFallbackToMainThread(error)) { throw error; } this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); this._throwIfDisposed(); const result = await this._runtime.loadModelFromUrl(url, { ...options, signal: this._operationSignal() || options.signal || null }); this._rememberLoadedModel(url, options); return result; } } async prefetchModelToCache(url, options = {}) { this._throwIfDisposed(); if (!this._runtime) { this._runtime = this._createRuntime(); } try { return await this._runtime.prefetchModelToCache(url, options); } finally { this._refreshCacheMetadataSnapshot(); } } async evictModelFromCache(url, options = {}) { this._throwIfDisposed(); if (!this._runtime) { this._runtime = this._createRuntime(); } try { return await this._runtime.evictModelFromCache(url, options); } finally { this._refreshCacheMetadataSnapshot(); } } async createCompletion(prompt, options = {}) { resolveCompletionSamplingOptions(options); const onUsage = options?.onUsage; if (typeof onUsage !== "function") { return this._runExclusive( () => this._createCompletionUnlocked(prompt, options), { signal: options?.signal, abortMessage: "Generation was cancelled.", kind: "generation" } ); } let usage = null; let text; try { text = await this._runExclusive( () => this._createCompletionUnlocked(prompt, { ...options, onUsage: (value) => { usage = value; } }), { signal: options.signal, abortMessage: "Generation was cancelled.", kind: "generation" } ); } catch (error) { if (usage != null && error?.name === "AbortError") { onUsage(usage); } throw error; } if (usage != null) { onUsage(usage); } return text; } async getCompletionCapabilities() { return this._runExclusive( () => this._getCompletionCapabilitiesUnlocked(), { kind: "completion-capabilities" } ); } async _getCompletionCapabilitiesUnlocked() { if (!this._workerProxy) { return this._runtime?.getCompletionCapabilities() ?? { ...NO_COMPLETION_CAPABILITIES }; } try { await this._restoreWorkerModelIfMissing(); return await this._callWorker("getCompletionCapabilities", []); } catch (error) { this._throwIfOperationCancelled(error, "Completion capability probe was cancelled."); if (!this._isWorkerUnusableError(error)) { throw error; } this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback({}, error); return this._runtime.getCompletionCapabilities(); } } async _createCompletionUnlocked(prompt, options = {}) { const isWarmup = options?.warmup === true; const hasRetriedEmptyMultimodal = options?.__llamadartEmptyRetryAttempted === true; const workerAllowed = this._config?.disableWorker !== true; if (this._hasMediaParts(options) && workerAllowed) { const hasWorkerFallback = typeof this._workerFallbackReason === "string" && this._workerFallbackReason.length > 0; if (hasWorkerFallback && !this._workerProxy && !this._isCpuModelMode()) { await this._ensureRuntimeReadyAfterWorkerFallback(options, null); return this._runtime.createCompletion(prompt, options); } try { if (!this._workerProxy) { await this._replaceWorkerProxyForMultimodalCpuMode(); } await this._ensureWorkerMultimodalCpuMode(); } catch (error) { this._throwIfOperationCancelled(error, "Generation was cancelled."); const reason = serializeWorkerError(error); if (isWarmup) { this._emitBridgeWarn( `llamadart: multimodal warmup skipped after worker setup issue (${reason}).` ); return ""; } this._emitBridgeWarn( `llamadart: unable to prepare multimodal worker CPU mode (${reason}).` ); if (this._isCpuModelMode()) { throw new Error( `CPU multimodal worker setup failed (${reason}). Reload model and retry with a smaller image.` ); } this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback(options, error); return this._runtime.createCompletion(prompt, options); } } if (!this._workerProxy) { return this._runtime.createCompletion(prompt, options); } try { await this._restoreWorkerModelIfMissing(); const workerOptions = { ...options }; delete workerOptions.onToken; delete workerOptions.onUsage; delete workerOptions.signal; delete workerOptions.__llamadartEmptyRetryAttempted; const speculative = options.speculativeDecoding; if (speculative != null && typeof speculative === "object" && !Array.isArray(speculative)) { const resolveCache = (source) => typeof source === "string" && source.length > 0 ? normalizeAbsoluteUrl(source) : source; workerOptions.speculativeDecoding = { ...speculative, ngramCacheStatic: resolveCache(speculative.ngramCacheStatic), ngramCacheDynamic: resolveCache(speculative.ngramCacheDynamic) }; } const stallTimeoutMs = this._workerCompletionStallTimeoutMs(options); let timeoutHandle = null; let rejectOnStall = null; const clearStallTimer = () => { if (timeoutHandle != null) { globalThis.clearTimeout(timeoutHandle); timeoutHandle = null; } }; const armStallTimer = () => { clearStallTimer(); if (!Number.isFinite(stallTimeoutMs) || stallTimeoutMs <= 0) { return; } timeoutHandle = globalThis.setTimeout(() => { const timeoutError = new Error( `Bridge worker completion stalled for ${stallTimeoutMs}ms.` ); timeoutError.llamadartWorkerTimeout = true; this._cancelOperation(this._activeOperation, "worker-timeout"); rejectOnStall?.(timeoutError); }, stallTimeoutMs); }; const stallPromise = new Promise((_, reject) => { rejectOnStall = reject; }); armStallTimer(); let sawWorkerTokenEvent = false; try { const workerResult = await Promise.race([ this._callWorker( "createCompletion", [prompt, workerOptions], (event) => { if (event.event === "usage") { if (typeof options.onUsage === "function") { options.onUsage(event.payload); } return; } if (event.event !== "token") { return; } armStallTimer(); sawWorkerTokenEvent = true; if (typeof options.onToken !== "function") { return; } const payload = event.payload || {}; const piece = typeof payload.pieceText === "string" ? payload.pieceText : Uint8Array.from(Array.isArray(payload.piece) ? payload.piece : []); options.onToken(piece, String(payload.currentText || "")); } ), stallPromise ]); if (this._hasMediaParts(options) && !isWarmup && !sawWorkerTokenEvent && String(workerResult || "").trim().length == 0) { this._emitBridgeWarn( "llamadart: multimodal worker produced empty response without token events." ); if (!hasRetriedEmptyMultimodal) { this._emitBridgeWarn( "llamadart: retrying multimodal worker once after empty response." ); try { await this._replaceWorkerProxyForMultimodalCpuMode(); await this._ensureWorkerMultimodalCpuMode(); } catch (retrySetupError) { this._emitBridgeWarn( `llamadart: multimodal empty-response retry setup failed (${serializeWorkerError(retrySetupError)}).` ); } return this._createCompletionUnlocked(prompt, { ...options, __llamadartEmptyRetryAttempted: true }); } } this._throwIfCallerCancelled("Generation was cancelled."); return workerResult; } finally { clearStallTimer(); } } catch (error) { this._throwIfOperationCancelled(error, "Generation was cancelled."); if (serializeWorkerError(error).includes(INVALID_GRAMMAR_ERROR_TEXT)) { throw error; } if (this._hasMediaParts(options)) { const reason = serializeWorkerError(error); if (isWarmup) { this._emitBridgeWarn( `llamadart: multimodal warmup skipped after worker request issue (${reason}).` ); return ""; } if (this._isCpuModelMode()) { this._emitBridgeWarn( `llamadart: CPU multimodal worker request failed (${reason}); skipping main-thread fallback.` ); throw new Error( `CPU multimodal request failed (${reason}). Reload model and retry with a smaller image.` ); } this._emitBridgeWarn( `llamadart: multimodal worker request failed (${reason}); falling back to main-thread runtime.` ); this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback(options, error); return this._runtime.createCompletion(prompt, options); } if (!this._isWorkerUnusableError(error)) { throw error; } this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback(options, error); return this._runtime.createCompletion(prompt, options); } } async loadMultimodalProjector(url) { return this._runExclusive( () => this._loadMultimodalProjectorUnlocked(url), { kind: "projector-load" } ); } async _loadMultimodalProjectorUnlocked(url) { const startedWithWorker = this._workerProxy != null; const invokeRuntimeLoad = async () => { if (!this._runtime) { this._runtime = this._createRuntime(); } await this._ensureRuntimeReadyAfterWorkerFallback({}, null); const result = await this._runtime.loadMultimodalProjector(url); this._rememberLoadedMmProj(url); this._supportsVision = this._runtime.supportsVision(); this._supportsAudio = this._runtime.supportsAudio(); return result; }; try { if (!this._workerProxy) { return await invokeRuntimeLoad(); } await this._ensureWorkerMultimodalCpuMode(); const result = await this._callWorker("loadMultimodalProjector", [url]); this._rememberLoadedMmProj(url); return result; } catch (error) { this._throwIfOperationCancelled(error, "Projector operation was cancelled."); if (!startedWithWorker) { throw error; } const reason = serializeWorkerError(error); this._emitBridgeWarn( `llamadart: multimodal worker setup failed (${reason}).` ); if (this._isCpuModelMode()) { try { await this._replaceWorkerProxyForMultimodalCpuMode(); await this._ensureWorkerMultimodalCpuMode(); const retryResult = await this._callWorker("loadMultimodalProjector", [url]); this._rememberLoadedMmProj(url); this._emitBridgeWarn( "llamadart: CPU multimodal worker setup recovered after worker restart." ); return retryResult; } catch (retryError) { const retryReason = serializeWorkerError(retryError); throw new Error( `CPU multimodal projector setup failed (${retryReason}). Reload model and retry with a smaller image.` ); } } this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); return invokeRuntimeLoad(); } } async unloadMultimodalProjector() { return this._runExclusive( () => this._unloadMultimodalProjectorUnlocked(), { kind: "projector-unload" } ); } async _unloadMultimodalProjectorUnlocked() { if (!this._workerProxy) { const result = await this._runtime.unloadMultimodalProjector(); this._loadedMmProjUrl = null; this._supportsVision = this._runtime.supportsVision(); this._supportsAudio = this._runtime.supportsAudio(); return result; } try { const result = await this._callWorker("unloadMultimodalProjector", []); this._loadedMmProjUrl = null; return result; } catch (error) { this._throwIfOperationCancelled(error, "Projector operation was cancelled."); this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); const result = await this._runtime.unloadMultimodalProjector(); this._loadedMmProjUrl = null; this._supportsVision = this._runtime.supportsVision(); this._supportsAudio = this._runtime.supportsAudio(); return result; } } // Loads the remembered draft model into the direct runtime when that // runtime does not hold it. async _ensureRuntimeDraftModel() { const draft = this._loadedDraftModel; const runtime = this._runtime; if (!draft || !runtime || runtime._draftModel?.url === draft.url) { return; } this._throwIfDisposed(); try { await runtime.loadDraftModel(draft.url, { ...draft.options, signal: this._operationSignal() || void 0 }); } catch (error) { this._throwIfOperationCancelled(error, "Draft model reload was cancelled."); this._forgetDraftModelAfterReloadFailure(error); } this._throwIfDisposed(); } // A reloaded target starts without its draft model. A failed draft reload // leaves none, so draft-* strategies then reject until loadDraftModel runs. _forgetDraftModelAfterReloadFailure(error) { this._loadedDraftModel = null; this._emitBridgeWarn( `llamadart: draft model reload failed after the model was reloaded (${serializeWorkerError(error)}); call loadDraftModel again.` ); } async loadDraftModel(url, options = {}) { return this._runExclusive( () => this._loadDraftModelUnlocked(url, options), { signal: options?.signal, abortMessage: "Draft model load was cancelled.", kind: "draft-model-load" } ); } async _loadDraftModelUnlocked(url, options = {}) { const absoluteUrl = typeof url === "string" && url.length > 0 ? normalizeAbsoluteUrl(url) : url; const remembered = { url: absoluteUrl, options: typeof options?.useCache === "boolean" ? { useCache: options.useCache } : {} }; const loadInRuntime = async () => { this._loadedDraftModel = null; const info = await this._runtime.loadDraftModel(absoluteUrl, { ...options, signal: this._operationSignal() || options.signal || void 0 }); this._loadedDraftModel = remembered; return info; }; if (!this._workerProxy) { return loadInRuntime(); } const workerOptions = { ...options }; delete workerOptions.progressCallback; delete workerOptions.signal; try { await this._restoreWorkerModelIfMissing(); this._loadedDraftModel = null; const info = await this._callWorker( "loadDraftModel", [absoluteUrl, workerOptions], (event) => { if (event.event === "progress" && typeof options.progressCallback === "function") { options.progressCallback(event.payload || {}); } } ); this._throwIfCallerCancelled("Draft model load was cancelled."); this._loadedDraftModel = remembered; return info; } catch (error) { this._throwIfOperationCancelled(error, "Draft model load was cancelled."); if (!this._isWorkerUnusableError(error)) { throw error; } this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback({}, error); return loadInRuntime(); } } async unloadDraftModel() { return this._runExclusive( () => this._unloadDraftModelUnlocked(), { kind: "draft-model-unload" } ); } async _unloadDraftModelUnlocked() { if (!this._workerProxy) { await this._runtime?.unloadDraftModel(); this._loadedDraftModel = null; return; } try { await this._callWorker("unloadDraftModel", []); this._loadedDraftModel = null; } catch (error) { this._throwIfOperationCancelled(error, "Draft model unload was cancelled."); if (!this._isWorkerUnusableError(error)) { throw error; } this._loadedDraftModel = null; this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._runtime?.unloadDraftModel(); } } supportsVision() { return this._supportsVision; } supportsAudio() { return this._supportsAudio; } async getTextToSpeechCapabilities() { return this._runExclusive( () => this._getTextToSpeechCapabilitiesUnlocked(), { kind: "tts-capabilities" } ); } async _getTextToSpeechCapabilitiesUnlocked() { if (!this._workerProxy) { return this._runtime.getTextToSpeechCapabilities(); } try { await this._restoreWorkerModelIfMissing(); return await this._callWorker("getTextToSpeechCapabilities", []); } catch (error) { this._throwIfOperationCancelled(error, "Text-to-speech capability probe was cancelled."); this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback({}, error); return this._runtime.getTextToSpeechCapabilities(); } } async synthesizeSpeech(options = {}) { return this._runExclusive( () => this._synthesizeSpeechUnlocked(options), { signal: options?.signal, abortMessage: "Text-to-speech synthesis was cancelled.", kind: "text-to-speech" } ); } async _synthesizeSpeechUnlocked(options = {}) { if (options.signal?.aborted) { throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); } if (!this._workerProxy) { return this._runtime.synthesizeSpeech(options); } try { await this._restoreWorkerModelIfMissing(); const workerOptions = { ...options }; delete workerOptions.onProgress; delete workerOptions.signal; const transferList = []; const speakerBytes = toUint8Array(options.speakerAudio); if (speakerBytes && speakerBytes.length > 0) { const copy = new Uint8Array(speakerBytes); workerOptions.speakerAudio = copy; transferList.push(copy.buffer); } const result = await this._callWorker( "synthesizeSpeech", [workerOptions], (event) => { if (event.event === "progress") { options.onProgress?.(event.payload || {}); } }, transferList ); this._throwIfCallerCancelled("Text-to-speech synthesis was cancelled."); return result; } catch (error) { this._throwIfOperationCancelled(error, "Text-to-speech synthesis was cancelled."); if (error?.name === "AbortError" || options.signal?.aborted || serializeWorkerError(error).toLowerCase().includes("cancel")) { throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); } const canRetryOnCpu = !this._isCpuModelMode() && (this._isDispatchWorkgroupLimitError(error) || this._isWorkerRequestTimeoutError(error)); if (canRetryOnCpu) { this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback( { _llamadartForceRuntimeReload: true, _llamadartTextToSpeech: true }, error ); if (options.signal?.aborted) { throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); } return this._runtime.synthesizeSpeech(options); } if (this._isWorkerRequestTimeoutError(error)) { this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); } throw error; } } _decisionHeadMap() { if (!(this._decisionHeads instanceof Map)) { this._decisionHeads = /* @__PURE__ */ new Map(); this._nextDecisionHandle = 1; } return this._decisionHeads; } _registerDecisionHead(owner, info) { const heads = this._decisionHeadMap(); const handle = this._nextDecisionHandle++; heads.set(handle, { owner, handle: Number(info?.handle) }); return { ...info, handle }; } _decisionOwner() { return this._workerProxy || this._runtime || null; } _staleDecisionHeadError(handle) { return new Error( `Decision head ${handle} is not loaded; it was freed, its model was unloaded, or the bridge runtime restarted. Load the decision head again.` ); } _resolveDecisionHead(handle) { decisionHandleFrom(handle); const heads = this._decisionHeadMap(); const entry = heads.get(handle); if (!entry) { return null; } if (entry.owner !== this._decisionOwner()) { heads.delete(handle); return null; } return entry; } _requireDecisionRuntime() { if (!this._runtime) { throw new Error("No model loaded. Call loadModelFromUrl first."); } return this._runtime; } /** * Moves a failed worker's work to the main-thread runtime. Decision heads * the worker held are gone with it; the model, its projector and the applied * LoRA adapters are restored so later calls keep working. */ async _recoverWorkerOnMainThread(error) { this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback({}, error); } async getDecisionCapabilities() { return this._runExclusive( () => this._getDecisionCapabilitiesUnlocked(), { kind: "decision-capabilities" } ); } async _getDecisionCapabilitiesUnlocked() { if (!this._workerProxy) { return this._runtime ? this._runtime.getDecisionCapabilities() : { apiVersion: DECISION_API_VERSION, supported: false, reason: "WebGPU core is not initialized" }; } try { return await this._callWorker("getDecisionCapabilities", []); } catch (error) { this._throwIfOperationCancelled(error, "Decision capability probe was cancelled."); if (!this._shouldFallbackToMainThread(error)) { throw error; } await this._recoverWorkerOnMainThread(error); return this._requireDecisionRuntime().getDecisionCapabilities(); } } async loadDecisionHead(source, options = {}) { return this._runExclusive( () => this._loadDecisionHeadUnlocked(source, options), { kind: "decision-head-load" } ); } async _loadDecisionHeadUnlocked(source, options = {}) { const loadInRuntime = async () => { const runtime = this._requireDecisionRuntime(); return this._registerDecisionHead( runtime, await runtime.loadDecisionHead(source, options) ); }; if (!this._workerProxy) { return loadInRuntime(); } const workerOptions = { configJson: options?.configJson ?? null }; let workerSource = source; const transferList = []; if (typeof source === "string") { workerSource = source.length > 0 ? normalizeAbsoluteUrl(source) : source; } else { const bytes = bufferSourceBytes(source); if (bytes) { const copy = new Uint8Array(bytes); workerSource = copy; transferList.push(copy.buffer); } } try { await this._restoreWorkerModelIfMissing(); const proxy = this._workerProxy; const info = await this._callWorker( "loadDecisionHead", [workerSource, workerOptions], (event) => { if (event.event === "progress") { options?.onProgress?.(event.payload || {}); } }, transferList ); return this._registerDecisionHead(proxy, info); } catch (error) { this._throwIfOperationCancelled(error, "Decision head load was cancelled."); if (!this._shouldFallbackToMainThread(error)) { throw error; } await this._recoverWorkerOnMainThread(error); return loadInRuntime(); } } async runDecision(handle, sequences) { return this._runExclusive( () => this._runDecisionUnlocked(handle, sequences), { kind: "decision-run" } ); } async _runDecisionUnlocked(handle, sequences) { const entry = this._resolveDecisionHead(handle); if (!entry) { throw this._staleDecisionHeadError(handle); } const normalized = normalizeDecisionSequences(sequences); if (!this._workerProxy) { return this._requireDecisionRuntime().runDecision(entry.handle, normalized); } try { return await this._callWorker("runDecision", [entry.handle, normalized]); } catch (error) { this._throwIfOperationCancelled(error, "Decision run was cancelled."); if (!this._shouldFallbackToMainThread(error)) { throw error; } this._decisionHeadMap().delete(handle); await this._recoverWorkerOnMainThread(error); throw new Error( `Decision head ${handle} was lost when the bridge worker failed (${serializeWorkerError(error)}). Load the decision head again.` ); } } async freeDecisionHead(handle) { return this._runExclusive( () => this._freeDecisionHeadUnlocked(handle), { kind: "decision-head-free" } ); } async _freeDecisionHeadUnlocked(handle) { const entry = this._resolveDecisionHead(handle); if (!entry) { return; } this._decisionHeadMap().delete(handle); if (!this._workerProxy) { await this._requireDecisionRuntime().freeDecisionHead(entry.handle); return; } try { await this._callWorker("freeDecisionHead", [entry.handle]); } catch (error) { this._throwIfOperationCancelled(error, "Decision head free was cancelled."); if (!this._shouldFallbackToMainThread(error)) { throw error; } await this._recoverWorkerOnMainThread(error); } } // Test doubles skip the constructor, so the LoRA state starts on first use. _loraAdapterMap() { if (!(this._loraAdapters instanceof Map)) { this._loraAdapters = /* @__PURE__ */ new Map(); this._activeLoraScales = /* @__PURE__ */ new Map(); this._nextLoraHandle = 1; } return this._loraAdapters; } _activeLoraScaleMap() { this._loraAdapterMap(); return this._activeLoraScales; } _forgetLoraAdapters() { this._loraAdapterMap().clear(); this._activeLoraScaleMap().clear(); } /** * Marks the adapters `owner` holds as lost before it loads the model again, * which frees them. An adapter with a retained source reloads on the next * restore or set; one without is forgotten. */ _dropLoraAdaptersOf(owner) { for (const [handle, entry] of this._loraAdapterMap()) { if (owner === null || entry.owner !== owner) { continue; } if (entry.source === null) { this._loraAdapterMap().delete(handle); this._activeLoraScaleMap().delete(handle); } else { entry.owner = null; } } } _loraOwner() { return this._workerProxy || this._runtime || null; } _requireLoraRuntime() { if (!this._runtime) { throw new Error("No model loaded. Call loadModelFromUrl first."); } return this._runtime; } _loraAdapterEntry(handle) { const entry = this._loraAdapterMap().get(handle); if (!entry) { throw staleLoraAdapterError(handle); } return entry; } // Loads an adapter into the current owner: the worker, or the direct runtime. async _loadLoraAdapterIntoOwner(source, options) { if (!this._workerProxy) { const runtime = this._requireLoraRuntime(); const info2 = await runtime.loadLoraAdapter(source, { ...options, signal: this._operationSignal() || options.signal || null }); return { owner: runtime, handle: info2.handle }; } const workerOptions = {}; if (options.useCache !== void 0) { workerOptions.useCache = options.useCache; } if (options.cacheName !== void 0) { workerOptions.cacheName = options.cacheName; } let workerSource = source; const transferList = []; if (typeof source !== "string") { workerSource = new Uint8Array(source); transferList.push(workerSource.buffer); } const proxy = this._workerProxy; const info = await this._callWorker( "loadLoraAdapter", [workerSource, workerOptions], (event) => { if (event.event === "progress") { options.progressCallback?.(event.payload || {}); } }, transferList ); return { owner: proxy, handle: Number(info?.handle) }; } // Reloads an adapter its owner lost, into the current owner. async _ensureLoraAdapterOnOwner(handle, entry) { if (entry.owner === this._loraOwner()) { return; } if (entry.source === null) { this._loraAdapterMap().delete(handle); this._activeLoraScaleMap().delete(handle); throw staleLoraAdapterError(handle); } const loaded = await this._loadLoraAdapterIntoOwner(entry.source, entry.cacheOptions); entry.owner = loaded.owner; entry.handle = loaded.handle; } async _callLoraOwner(method, args) { if (this._workerProxy) { await this._callWorker(method, args); return; } const runtime = this._requireLoraRuntime(); if (method === "setLoraAdapter") { await runtime.setLoraAdapter(args[0], args[1]); } else if (method === "removeLoraAdapter") { await runtime.removeLoraAdapter(args[0]); } else { await runtime.clearLoraAdapters(); } } /** * Makes the current owner apply exactly the facade's active adapters after a * worker restart or a fallback to the main thread. The owner's own set is * cleared first, since a runtime that held the model earlier may still apply * adapters from then. Inactive adapters reload when next set. */ async _restoreLoraAdapters() { const owner = this._loraOwner(); if (![...this._loraAdapterMap().values()].some((entry) => entry.owner !== owner)) { return; } await this._callLoraOwner("clearLoraAdapters", []); for (const [handle, scale] of [...this._activeLoraScaleMap()]) { const entry = this._loraAdapterEntry(handle); await this._ensureLoraAdapterOnOwner(handle, entry); await this._callLoraOwner("setLoraAdapter", [entry.handle, scale]); } } // Runs a LoRA call on the current owner, moving to the main-thread runtime // and running it there once if the worker fails. async _runLoraCall(call, cancelMessage) { if (!this._workerProxy) { return call(); } try { await this._restoreWorkerModelIfMissing(); return await call(); } catch (error) { this._throwIfOperationCancelled(error, cancelMessage); if (!this._shouldFallbackToMainThread(error)) { throw error; } await this._recoverWorkerOnMainThread(error); return call(); } } async getLoraAdapterCapabilities() { return this._runExclusive( () => this._getLoraAdapterCapabilitiesUnlocked(), { kind: "lora-capabilities" } ); } async _getLoraAdapterCapabilitiesUnlocked() { if (!this._workerProxy && !this._runtime) { return { apiVersion: LORA_API_VERSION, supported: false, reason: "WebGPU core is not initialized" }; } return this._runLoraCall( () => this._workerProxy ? this._callWorker("getLoraAdapterCapabilities", []) : Promise.resolve(this._requireLoraRuntime().getLoraAdapterCapabilities()), "LoRA capability probe was cancelled." ); } async loadLoraAdapter(source, options = {}) { return this._runExclusive( () => this._loadLoraAdapterUnlocked(source, options || {}), { signal: options?.signal, abortMessage: LORA_LOAD_ABORT_MESSAGE, kind: "lora-adapter-load" } ); } async _loadLoraAdapterUnlocked(source, options) { let normalized; if (typeof source === "string") { if (source.length === 0) { throw new Error("LoRA adapter URL is empty."); } normalized = normalizeAbsoluteUrl(source); } else { const bytes = bufferSourceBytes(source); if (!bytes) { throw new TypeError("LoRA adapter source must be a URL string, an ArrayBuffer or a typed array."); } if (bytes.byteLength === 0) { throw new Error("LoRA adapter bytes are empty."); } normalized = bytes; } let retained = null; if (this._config?.disableWorker !== true) { retained = typeof normalized === "string" ? normalized : new Uint8Array(normalized); } const loaded = await this._runLoraCall( () => this._loadLoraAdapterIntoOwner(retained ?? normalized, options), LORA_LOAD_ABORT_MESSAGE ); this._loraAdapterMap(); const handle = this._nextLoraHandle++; this._loraAdapterMap().set(handle, { owner: loaded.owner, handle: loaded.handle, source: retained, cacheOptions: { useCache: options.useCache, cacheName: options.cacheName } }); return { handle }; } async setLoraAdapter(handle, scale = 1) { const facadeHandle = loraHandleFrom(handle); const checkedScale = loraScaleFrom(scale); return this._runExclusive( () => this._setLoraAdapterUnlocked(facadeHandle, checkedScale), { kind: "lora-adapter-set" } ); } async _setLoraAdapterUnlocked(handle, scale) { const entry = this._loraAdapterEntry(handle); await this._runLoraCall(async () => { await this._ensureLoraAdapterOnOwner(handle, entry); await this._callLoraOwner("setLoraAdapter", [entry.handle, scale]); }, "LoRA adapter update was cancelled."); this._activeLoraScaleMap().set(handle, scale); } async removeLoraAdapter(handle) { const facadeHandle = loraHandleFrom(handle); return this._runExclusive( () => this._removeLoraAdapterUnlocked(facadeHandle), { kind: "lora-adapter-remove" } ); } async _removeLoraAdapterUnlocked(handle) { const entry = this._loraAdapterEntry(handle); await this._runLoraCall(async () => { if (entry.owner === this._loraOwner()) { await this._callLoraOwner("removeLoraAdapter", [entry.handle]); } }, "LoRA adapter update was cancelled."); this._activeLoraScaleMap().delete(handle); } async clearLoraAdapters() { return this._runExclusive( () => this._clearLoraAdaptersUnlocked(), { kind: "lora-adapters-clear" } ); } async _clearLoraAdaptersUnlocked() { await this._runLoraCall( () => this._callLoraOwner("clearLoraAdapters", []), "LoRA adapter update was cancelled." ); this._activeLoraScaleMap().clear(); } async tokenize(text, addSpecial = true) { return this._runExclusive( () => this._tokenizeUnlocked(text, addSpecial), { kind: "tokenize" } ); } async _tokenizeUnlocked(text, addSpecial = true) { if (!this._workerProxy) { return this._runtime.tokenize(text, addSpecial); } try { await this._restoreWorkerModelIfMissing(); return await this._callWorker("tokenize", [text, addSpecial]); } catch (error) { this._throwIfOperationCancelled(error, "Tokenization was cancelled."); if (!this._isWorkerUnusableError(error)) { throw error; } this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback({}, error); return this._runtime.tokenize(text, addSpecial); } } async stateSaveFile(path, tokens = []) { return this._runExclusive( () => this._stateSaveFileUnlocked(path, tokens), { kind: "state-save-file" } ); } async _stateSaveFileUnlocked(path, tokens = []) { if (!this._workerProxy) { return this._runtime.stateSaveFile(path, tokens); } const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); await this._restoreWorkerModelIfMissing(); return this._callWorker("stateSaveFile", [path, normalized]); } // The capacity default is resolved inside the slot: evaluating it as a // parameter default would read runtime state before the queue was taken. async stateLoadFile(path, tokenCapacity = void 0) { return this._runExclusive( () => this._stateLoadFileUnlocked(path, tokenCapacity), { kind: "state-load-file" } ); } _resolveTokenCapacity(tokenCapacity) { let numericCapacity; try { numericCapacity = Number(tokenCapacity); } catch { return this.getContextSize(); } return numericCapacity > 0 ? Math.trunc(numericCapacity) : this.getContextSize(); } async _stateLoadFileUnlocked(path, tokenCapacity) { tokenCapacity = this._resolveTokenCapacity(tokenCapacity); if (!this._workerProxy) { return this._runtime.stateLoadFile(path, tokenCapacity); } await this._restoreWorkerModelIfMissing(); return this._callWorker("stateLoadFile", [path, tokenCapacity]); } async stateSaveBytes(tokens = []) { return this._runExclusive( () => this._stateSaveBytesUnlocked(tokens), { kind: "state-save-bytes" } ); } async _stateSaveBytesUnlocked(tokens = []) { if (!this._workerProxy) { return this._runtime.stateSaveBytes(tokens); } const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); await this._restoreWorkerModelIfMissing(); return this._callWorker("stateSaveBytes", [normalized]); } async stateLoadBytes(bytes, tokenCapacity = void 0) { return this._runExclusive( () => this._stateLoadBytesUnlocked(bytes, tokenCapacity), { kind: "state-load-bytes" } ); } async _stateLoadBytesUnlocked(bytes, tokenCapacity) { tokenCapacity = this._resolveTokenCapacity(tokenCapacity); if (!this._workerProxy) { return this._runtime.stateLoadBytes(bytes, tokenCapacity); } const normalizedBytes = toUint8Array(bytes); if (!normalizedBytes || normalizedBytes.length === 0) { throw new Error("State bytes are empty."); } await this._restoreWorkerModelIfMissing(); const transferableBytes = new Uint8Array(normalizedBytes); return this._callWorker( "stateLoadBytes", [transferableBytes, tokenCapacity], null, [transferableBytes.buffer] ); } async detokenize(tokens, special = false) { return this._runExclusive( () => this._detokenizeUnlocked(tokens, special), { kind: "detokenize" } ); } async _detokenizeUnlocked(tokens, special = false) { if (!this._workerProxy) { return this._runtime.detokenize(tokens, special); } const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); try { await this._restoreWorkerModelIfMissing(); return await this._callWorker("detokenize", [normalized, special]); } catch (error) { this._throwIfOperationCancelled(error, "Detokenization was cancelled."); if (!this._isWorkerUnusableError(error)) { throw error; } this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback({}, error); return this._runtime.detokenize(normalized, special); } } async getEmbeddingCapabilities() { return this._runExclusive(async () => { try { return await this._getEmbeddingCapabilitiesUnlocked(); } catch (error) { this._throwIfOperationCancelled(error, "Embedding capability probe was cancelled."); if (!this._workerProxy || !this._isWorkerUnusableError(error)) throw error; await this._recoverEmbeddingWorker(error); return this._runtime.getEmbeddingCapabilities(); } }, { kind: "embedding-capabilities" }); } async _getEmbeddingCapabilitiesUnlocked() { if (!this._workerProxy) { await this._ensureRememberedEmbeddingProjector(); return this._runtime.getEmbeddingCapabilities(); } try { await this._restoreWorkerModelIfMissing(); return embeddingCapabilitiesFrom( await this._callWorker("getEmbeddingCapabilities", []), embeddingAssetVersion(this._config.workerUrl) ); } catch (error) { this._throwIfOperationCancelled(error, "Embedding capability probe was cancelled."); if (error instanceof Error && /bridge\[method\] is not a function/.test(error.message)) { throw missingEmbeddingCapability(embeddingAssetVersion(this._config.workerUrl)); } throw error; } } async embed(text, options = {}) { const input = typeof text === "string" ? text : snapshotEmbedInput(text); return this._runExclusive( () => this._embedUnlocked(input, options), { kind: "embedding" } ); } _hasEmbeddingMediaParts(input) { return Array.isArray(input) && input.some((part) => part.type !== "text"); } async _recoverEmbeddingWorker(error) { this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback({}, error); await this._ensureRememberedEmbeddingProjector(); } async _ensureRememberedEmbeddingProjector() { if (!this._loadedMmProjUrl || !this._runtime?._modelBytes || this._runtime._mmProjPath) return; this._throwIfDisposed(); try { await this._runtime.loadMultimodalProjector(this._loadedMmProjUrl); } catch (error) { this._throwIfOperationCancelled(error, "Embedding projector restore was cancelled."); this._emitBridgeWarn("llamadart: embedding projector restore failed; keeping the loaded model."); } this._throwIfDisposed(); } async _embedUnlocked(text, options = {}) { if (!this._workerProxy) { if (this._hasEmbeddingMediaParts(text)) await this._ensureRememberedEmbeddingProjector(); return this._runtime.embed(text, options); } try { await this._restoreWorkerModelIfMissing(); if (Array.isArray(text)) await this._getEmbeddingCapabilitiesUnlocked(); return await this._callWorker("embed", [text, options]); } catch (error) { this._throwIfOperationCancelled(error, "Embedding was cancelled."); if (!this._isWorkerUnusableError(error)) { throw error; } await this._recoverEmbeddingWorker(error); return this._runtime.embed(text, options); } } async embedBatch(texts, options = {}) { const inputs = Array.from(texts || [], (input) => typeof input === "string" ? input : snapshotEmbedInput(input)); return this._runExclusive( () => this._embedBatchUnlocked(inputs, options), { kind: "embedding-batch" } ); } async _embedBatchUnlocked(texts, options = {}) { const normalized = Array.isArray(texts) ? texts : Array.from(texts || []); if (!this._workerProxy) { if (normalized.some((input) => this._hasEmbeddingMediaParts(input))) await this._ensureRememberedEmbeddingProjector(); return this._runtime.embedBatch(normalized, options); } try { await this._restoreWorkerModelIfMissing(); if (normalized.some(Array.isArray)) await this._getEmbeddingCapabilitiesUnlocked(); return await this._callWorker("embedBatch", [normalized, options]); } catch (error) { this._throwIfOperationCancelled(error, "Batch embedding was cancelled."); if (!this._isWorkerUnusableError(error)) { throw error; } await this._recoverEmbeddingWorker(error); return this._runtime.embedBatch(normalized, options); } } async scoreNextToken(prompt, options = {}) { return this._runExclusive( () => this._scoreNextTokenUnlocked(prompt, options), { kind: "next-token-scoring" } ); } async _scoreNextTokenUnlocked(prompt, options = {}) { if (!this._workerProxy) { return this._runtime.scoreNextToken(prompt, options); } try { await this._restoreWorkerModelIfMissing(); return await this._callWorker("scoreNextToken", [prompt, options]); } catch (error) { this._throwIfOperationCancelled(error, "Next-token scoring was cancelled."); if (!this._isWorkerUnusableError(error)) { throw error; } this._disableWorkerFallback(error); await this._waitForWorkerDisposal(); await this._ensureRuntimeReadyAfterWorkerFallback({}, error); return this._runtime.scoreNextToken(prompt, options); } } getModelMetadata() { if (this._workerProxy) { return { ...this._metadata || {}, "llamadart.webgpu.execution": "worker" }; } const workerReason = typeof this._workerFallbackReason === "string" && this._workerFallbackReason.length > 0 ? this._workerFallbackReason : null; return { ...this._metadata || {}, "llamadart.webgpu.execution": "main-thread", ...workerReason == null ? {} : { "llamadart.webgpu.worker_fallback_reason": workerReason } }; } // The synchronous getters answer from the facade snapshot in both modes. // Reading the direct runtime here would ccall into the singleton core outside // the queue and race whichever async operation currently owns it; the snapshot // is refreshed by _captureDirectRuntimeState() while the slot is held. getContextSize() { return this._contextSize || 0; } isGpuActive() { return this._gpuActive; } getBackendName() { return this._backendName; } setLogLevel(level) { this._throwIfDisposed(); if (Number.isFinite(level)) { this._config.logLevel = Math.max(0, Math.min(4, Math.trunc(level))); } if (this._workerProxy) { this._callWorker("setLogLevel", [level]).catch(() => { }); if (this._runtime) { this._runtime.setLogLevel(level); } return; } this._runtime.setLogLevel(level); } cancel() { if (this._lifecycleState === "disposing" || this._lifecycleState === "disposed") { return; } if (this._activeOperation) { this._cancelOperation(this._activeOperation); return; } if (this._workerProxy) { this._sendWorkerCancel(this._workerProxy); } if (this._runtime) { try { this._runtime.cancel(); } catch (_) { } } } /** * Marks the bridge disposed synchronously, then tears down on the queue so * teardown never runs underneath an operation that still owns the runtime. * Work queued before disposal is rejected with a stable lifecycle error rather * than dereferencing a torn-down runtime. Not `async`, so repeated calls hand * back the identical teardown promise. */ dispose() { if (this._disposed) { return this._disposePromise; } this._disposed = true; this._lifecycleState = "disposing"; let resolveDispose = () => { }; let rejectDispose = () => { }; this._disposePromise = new Promise((resolve, reject) => { resolveDispose = resolve; rejectDispose = reject; }); this._notifyDisposalWaiters(); const teardown = this._runExclusive( () => this._disposeUnlocked(), { allowDisposed: true, kind: "dispose" } ); teardown.then( () => resolveDispose(), (error) => rejectDispose(error) ); return this._disposePromise; } async _disposeUnlocked() { const workerProxy = this._workerProxy; this._workerProxy = null; if (workerProxy) { this._retireWorkerProxy(workerProxy); } try { if (this._retiringWorkerDisposals?.size > 0) { await this._waitForWorkerDisposal(); } if (this._runtime) { const runtime = this._runtime; this._runtime = null; await runtime.dispose(); } } finally { this._metadata = {}; this._contextSize = 0; this._gpuActive = false; this._backendName = "WASM (Prototype bridge)"; this._supportsVision = false; this._supportsAudio = false; this._loadedModelUrl = null; this._loadedModelOptions = null; this._loadedMmProjUrl = null; this._loadedDraftModel = null; this._workerFallbackReason = null; this._multimodalWorkerCpuMode = false; this._workerModelMissing = false; this._decisionHeads?.clear(); this._forgetLoraAdapters(); this._lifecycleState = "disposed"; } } async applyChatTemplate(messages, addAssistant = true, customTemplate = null) { return this._runExclusive( () => this._applyChatTemplateUnlocked(messages, addAssistant, customTemplate), { kind: "template" } ); } async _applyChatTemplateUnlocked(messages, addAssistant = true, customTemplate = null) { if (!this._workerProxy) { return this._runtime.applyChatTemplate(messages, addAssistant, customTemplate); } try { return await this._callWorker("applyChatTemplate", [messages, addAssistant, customTemplate]); } catch (error) { this._throwIfOperationCancelled(error, "Chat template operation was cancelled."); if (!this._isWorkerUnusableError(error)) { throw error; } this._disableWorkerFallback(error); return this._runtime.applyChatTemplate(messages, addAssistant, customTemplate); } } }; // js/src/worker_host.ts var textDecoder = new TextDecoder(); var bridgeWorkerHostInstalled = false; function installBridgeWorkerHost() { if (bridgeWorkerHostInstalled) { return; } if (typeof self === "undefined" || typeof self.postMessage !== "function") { throw new Error("Bridge worker host can only run inside a worker context"); } bridgeWorkerHostInstalled = true; let bridge = null; const postError = (id, error) => { let state; try { state = bridge ? snapshotBridgeState(bridge) : void 0; } catch (_) { state = void 0; } self.postMessage({ type: "error", id, message: serializeWorkerError(error), ...error instanceof EmbeddingError ? { embeddingErrorCode: error.code } : {}, state }); }; self.onmessage = async (event) => { const message = event.data || {}; const type = message.type; const id = message.id ?? 0; try { if (type === "init") { bridge = new LlamaWebGpuBridge({ ...message.config || {}, disableWorker: true }); self.postMessage({ type: "ready" }); return; } if (type !== "call") { return; } if (!bridge) { throw new Error("Bridge worker is not initialized"); } const method = String(message.method || ""); const args = Array.isArray(message.args) ? message.args : []; if (method === "loadModelFromUrl") { const url = args[0]; const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; options.progressCallback = (progress) => { self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); }; const value2 = await bridge.loadModelFromUrl(url, options); self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) }); return; } if (method === "loadDraftModel") { const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; delete options.signal; options.progressCallback = (progress) => { self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); }; const value2 = await bridge.loadDraftModel(args[0], options); self.postMessage({ type: "result", id, value: value2 }); return; } if (method === "createCompletion") { const prompt = args[0]; const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; delete options.signal; let usage = null; options.onUsage = (value3) => { usage = value3; }; const tokenEventEncoding = typeof options.tokenEventEncoding === "string" ? String(options.tokenEventEncoding || "").toLowerCase() : "bytes"; const flushMsRaw = Number(options.tokenEventFlushMs); const tokenEventFlushMs = Number.isFinite(flushMsRaw) && flushMsRaw >= 0 ? Math.max(0, Math.min(200, Math.trunc(flushMsRaw))) : 0; const flushCharsRaw = Number(options.tokenEventFlushChars); const tokenEventFlushChars = Number.isFinite(flushCharsRaw) && flushCharsRaw > 0 ? Math.max(1, Math.min(1024, Math.trunc(flushCharsRaw))) : 0; const shouldEmitCurrentText = options.emitCurrentTextOnToken === true; let pendingPieceText = ""; let pendingPieceBytes = []; let pendingPieceByteLength = 0; let pendingPieceCharLength = 0; let pendingPieceCharDecoder = tokenEventFlushChars > 0 ? new TextDecoder() : null; let pendingCurrentText = ""; let flushTimer = null; const flushTokenTextPayload = () => { if (pendingPieceText.length === 0) { return; } self.postMessage({ type: "event", id, event: "token", payload: { pieceText: pendingPieceText, currentText: shouldEmitCurrentText ? pendingCurrentText : "" } }); pendingPieceText = ""; pendingCurrentText = ""; }; const flushTokenBytePayload = () => { if (pendingPieceByteLength === 0) { return; } const piece = new Uint8Array(pendingPieceByteLength); let offset = 0; for (const chunk of pendingPieceBytes) { piece.set(chunk, offset); offset += chunk.byteLength; } self.postMessage({ type: "event", id, event: "token", payload: { piece: Array.from(piece), currentText: shouldEmitCurrentText ? pendingCurrentText : "" } }); pendingPieceBytes = []; pendingPieceByteLength = 0; pendingPieceCharLength = 0; pendingCurrentText = ""; }; const flushTokenPayload = () => { if (tokenEventEncoding === "text") { flushTokenTextPayload(); return; } flushTokenBytePayload(); }; const scheduleTokenFlush = () => { if (tokenEventFlushMs <= 0 || flushTimer != null) { return; } flushTimer = globalThis.setTimeout(() => { flushTimer = null; flushTokenPayload(); }, tokenEventFlushMs); }; options.onToken = (piece, currentText) => { if (tokenEventEncoding === "text") { const pieceText = typeof piece === "string" ? piece : textDecoder.decode(toUint8Array(piece) || new Uint8Array()); if (pieceText.length === 0) { return; } if (tokenEventFlushMs > 0) { pendingPieceText += pieceText; if (shouldEmitCurrentText) { pendingCurrentText = String(currentText || ""); } if (tokenEventFlushChars > 0 && pendingPieceText.length >= tokenEventFlushChars) { if (flushTimer != null) { globalThis.clearTimeout(flushTimer); flushTimer = null; } flushTokenTextPayload(); return; } scheduleTokenFlush(); return; } self.postMessage({ type: "event", id, event: "token", payload: { pieceText, currentText: shouldEmitCurrentText ? String(currentText || "") : "" } }); return; } const normalizedPieceBytes = toUint8Array(piece) || new Uint8Array(); if (normalizedPieceBytes.byteLength === 0) { return; } if (tokenEventFlushMs > 0) { const pieceBytes = Array.isArray(piece) ? normalizedPieceBytes : Uint8Array.from(normalizedPieceBytes); pendingPieceBytes.push(pieceBytes); pendingPieceByteLength += pieceBytes.byteLength; if (pendingPieceCharDecoder != null) { pendingPieceCharLength += pendingPieceCharDecoder.decode( pieceBytes, { stream: true } ).length; } if (shouldEmitCurrentText) { pendingCurrentText = String(currentText || ""); } if (tokenEventFlushChars > 0 && pendingPieceCharLength >= tokenEventFlushChars) { if (flushTimer != null) { globalThis.clearTimeout(flushTimer); flushTimer = null; } flushTokenBytePayload(); return; } scheduleTokenFlush(); return; } self.postMessage({ type: "event", id, event: "token", payload: { piece: Array.from(normalizedPieceBytes), currentText: shouldEmitCurrentText ? String(currentText || "") : "" } }); }; let value2; try { value2 = await bridge.createCompletion(prompt, options); } finally { if (flushTimer != null) { globalThis.clearTimeout(flushTimer); flushTimer = null; } flushTokenPayload(); } flushTokenTextPayload(); if (usage != null) { self.postMessage({ type: "event", id, event: "usage", payload: usage }); } self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) }); return; } if (method === "loadMultimodalProjector") { const value2 = await bridge.loadMultimodalProjector(args[0]); self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) }); return; } if (method === "synthesizeSpeech") { const options = args[0] && typeof args[0] === "object" ? { ...args[0] } : {}; delete options.signal; options.onProgress = (progress) => { self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); }; const value2 = await bridge.synthesizeSpeech(options); const transfers = value2?.pcm?.buffer instanceof ArrayBuffer ? [value2.pcm.buffer] : []; self.postMessage( { type: "result", id, value: value2, state: snapshotBridgeState(bridge) }, transfers ); return; } if (method === "loadDecisionHead") { const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; options.onProgress = (progress) => { self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); }; const value2 = await bridge.loadDecisionHead(args[0], options); self.postMessage({ type: "result", id, value: value2 }); return; } if (method === "loadLoraAdapter") { const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; delete options.signal; options.progressCallback = (progress) => { self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); }; const value2 = await bridge.loadLoraAdapter(args[0], options); self.postMessage({ type: "result", id, value: value2 }); return; } if (method === "runDecision") { const value2 = await bridge.runDecision(args[0], args[1]); const transfers = []; for (const output of Array.isArray(value2) ? value2 : []) { for (const values of [output?.logits, output?.actLogits]) { const buffer = values?.buffer; if (buffer instanceof ArrayBuffer && !transfers.includes(buffer)) { transfers.push(buffer); } } } self.postMessage({ type: "result", id, value: value2 }, transfers); return; } if (method === "unloadMultimodalProjector") { const value2 = await bridge.unloadMultimodalProjector(); self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) }); return; } if (method === "stateSaveBytes") { const value2 = await bridge.stateSaveBytes(args[0]); const transfers = value2 && value2.buffer instanceof ArrayBuffer ? [value2.buffer] : []; self.postMessage({ type: "result", id, value: value2 }, transfers); return; } if (method === "stateLoadBytes") { const value2 = await bridge.stateLoadBytes(args[0], args[1]); self.postMessage({ type: "result", id, value: value2 }); return; } if (method === "dispose") { const value2 = await bridge.dispose(); self.postMessage({ type: "result", id, value: value2, state: emptyBridgeState() }); return; } const value = await bridge[method]( ...args || [] ); self.postMessage({ type: "result", id, value }); } catch (error) { postError(id, error); } }; } function shouldAutoBootBridgeWorkerHost() { if (typeof WorkerGlobalScope === "undefined" || !(globalThis instanceof WorkerGlobalScope)) { return false; } try { const href = String(globalThis.location?.href || ""); if (!href) { return false; } const url = new URL(href); return url.searchParams.get(bridgeWorkerModeParam) === "1"; } catch (_) { return false; } } function enableBridgeWorkerHost() { installBridgeWorkerHost(); } // js/src/llama_webgpu_bridge.js if (shouldAutoBootBridgeWorkerHost()) { installBridgeWorkerHost(); } if (typeof window !== "undefined") { const browserWindow = ( /** @type {Window & typeof globalThis & { LlamaWebGpuBridge?: typeof LlamaWebGpuBridge }} */ window ); if (!browserWindow.LlamaWebGpuBridge) { browserWindow.LlamaWebGpuBridge = LlamaWebGpuBridge; } } export { EmbeddingError, LlamaWebGpuBridge, enableBridgeWorkerHost };