Spaces:
Running
Running
Download webgpu_bridge/llama_webgpu_bridge.js from leehack/flutter-laya-tetris: direct link, hf CLI and curl.
- Browser
- Download file 296 kB
-
https://huggingface.co/spaces/leehack/flutter-laya-tetris/resolve/main/webgpu_bridge/llama_webgpu_bridge.js
- Command line
-
hf download hf://spaces/leehack/flutter-laya-tetris/webgpu_bridge/llama_webgpu_bridge.js
-
curl -L -o llama_webgpu_bridge.js https://huggingface.co/spaces/leehack/flutter-laya-tetris/resolve/main/webgpu_bridge/llama_webgpu_bridge.js
296 kB
| // Generated by scripts/build_js_bridge.mjs from js/src/llama_webgpu_bridge.js. | |
| // Do not edit this file directly; update js/src/ and run npm run build:js. | |
| // js/src/internal/abort.ts | |
| function createAbortError(message) { | |
| if (typeof DOMException === "function") { | |
| return new DOMException(message, "AbortError"); | |
| } | |
| const error = new Error(message); | |
| error.name = "AbortError"; | |
| return error; | |
| } | |
| function throwIfAborted(signal, message = "Bridge operation was cancelled.") { | |
| if (signal?.aborted) { | |
| throw createAbortError(message); | |
| } | |
| } | |
| // js/src/internal/embedding.ts | |
| var EmbeddingError = class extends Error { | |
| constructor(code, message) { | |
| super(message); | |
| this.name = "EmbeddingError"; | |
| this.code = code; | |
| } | |
| }; | |
| var EMPTY_EMBEDDING_CAPABILITIES = { | |
| apiVersion: 1, | |
| text: false, | |
| image: false, | |
| audio: false | |
| }; | |
| var EMBEDDING_MEDIA_MARKER = "<__media__>"; | |
| function embeddingAssetVersion(...sources) { | |
| for (const source of [...sources, import.meta.url]) { | |
| try { | |
| const version = new URL(source || "", import.meta.url).pathname.match(/(?:^|[/@])(v?\d+\.\d+\.\d+(?:-\d+)?)(?:[/]|$)/)?.[1]; | |
| if (version) return version; | |
| } catch (_) { | |
| } | |
| } | |
| return "unversioned/local"; | |
| } | |
| function missingEmbeddingCapability(version = embeddingAssetVersion()) { | |
| return new EmbeddingError( | |
| "EMBEDDING_CAPABILITY_UNAVAILABLE", | |
| `Loaded bridge/assets version ${version} has no known media-embedding capability API version 1. Upgrade the loaded bridge assets.` | |
| ); | |
| } | |
| function embeddingCapabilitiesFrom(value, version = embeddingAssetVersion()) { | |
| let parsed = value; | |
| if (typeof value === "string") { | |
| try { | |
| parsed = JSON.parse(value); | |
| } catch (_) { | |
| throw missingEmbeddingCapability(version); | |
| } | |
| } | |
| const caps = parsed; | |
| if (!caps || caps.apiVersion !== 1 || typeof caps.text !== "boolean" || typeof caps.image !== "boolean" || typeof caps.audio !== "boolean") { | |
| throw missingEmbeddingCapability(version); | |
| } | |
| return { apiVersion: 1, text: caps.text, image: caps.image, audio: caps.audio }; | |
| } | |
| function snapshotEmbedInput(input) { | |
| if (!Array.isArray(input)) return String(input); | |
| if (!Array.isArray(input) || input.length === 0) { | |
| throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding parts must be a nonempty array."); | |
| } | |
| const parts = Array.from(input, (part) => { | |
| if (!part || typeof part !== "object") { | |
| throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Invalid embedding part."); | |
| } | |
| if (part.type === "text") { | |
| if (typeof part.text !== "string") { | |
| throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding text part must contain a string."); | |
| } | |
| return { type: "text", text: part.text }; | |
| } | |
| if (part.type !== "image" && part.type !== "audio") { | |
| throw new EmbeddingError("EMBEDDING_MODALITY_UNSUPPORTED", "Embedding part modality is unsupported."); | |
| } | |
| if ("path" in part) { | |
| throw new EmbeddingError("EMBEDDING_MODALITY_UNSUPPORTED", "Browser embeddings require media bytes, URL or PCM; filesystem paths are unsupported."); | |
| } | |
| const sourceCount = Number(part.bytes != null) + Number(part.url != null) + Number("samples" in part && part.samples != null); | |
| if (sourceCount !== 1 || part.bytes != null && (!(part.bytes instanceof Uint8Array) || part.bytes.length === 0) || part.url != null && (typeof part.url !== "string" || part.url.length === 0)) { | |
| throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding media requires exactly one nonempty supported source."); | |
| } | |
| if ("samples" in part && part.samples != null) { | |
| if (part.type !== "audio" || !(part.samples instanceof Float32Array) || part.samples.length === 0 || !part.samples.every(Number.isFinite)) { | |
| throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding PCM must be finite mono 16 kHz Float32 samples."); | |
| } | |
| } | |
| return { ...part }; | |
| }); | |
| const merged = []; | |
| for (const part of parts) { | |
| const previous = merged.at(-1); | |
| if (part.type === "text" && previous?.type === "text") previous.text += part.text; | |
| else merged.push(part); | |
| } | |
| if (merged.some((part) => part.type === "text" && part.text.includes(EMBEDDING_MEDIA_MARKER))) { | |
| throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Reserved media marker in embedding text."); | |
| } | |
| return merged; | |
| } | |
| function requireEmbeddingModalities(input, caps, hasProjector) { | |
| if (typeof input === "string") return; | |
| if (!caps.text) { | |
| throw new EmbeddingError("MODEL_NOT_EMBEDDING", "Loaded model does not support pooled embeddings."); | |
| } | |
| for (const part of input) { | |
| if (part.type === "text") continue; | |
| if (!hasProjector) { | |
| throw new EmbeddingError("EMBEDDING_PROJECTOR_MISSING", "Media embeddings require a loaded projector."); | |
| } | |
| if (!caps[part.type]) { | |
| throw new EmbeddingError("EMBEDDING_MODALITY_UNSUPPORTED", "Loaded projector/model does not support the requested embedding modality."); | |
| } | |
| } | |
| } | |
| function embeddingCoreError(rc, version = embeddingAssetVersion()) { | |
| if (rc === -20) return missingEmbeddingCapability(version); | |
| const errors = { | |
| [-21]: ["MODEL_NOT_EMBEDDING", "Loaded model does not support this pooled embedding operation."], | |
| [-22]: ["EMBEDDING_PROJECTOR_MISSING", "Media embeddings require a loaded projector."], | |
| [-23]: ["EMBEDDING_INPUT_TOO_LARGE", "Expanded embedding input exceeds the allocated context, batch, sequence or single microbatch capacity."], | |
| [-24]: ["EMBEDDING_MODALITY_UNSUPPORTED", "Loaded projector does not support the requested embedding modality or PCM rate."] | |
| }; | |
| const [code, message] = errors[rc] || ["EMBEDDING_FAILED", "Embedding execution failed."]; | |
| return new EmbeddingError(code, message); | |
| } | |
| // js/src/internal/completion_options.ts | |
| var SPECULATIVE_DECODING_STRATEGIES = Object.freeze([ | |
| "draft-simple", | |
| "draft-eagle3", | |
| "draft-mtp", | |
| "draft-dflash", | |
| "draft-dspark", | |
| "ngram-simple", | |
| "ngram-map-k", | |
| "ngram-map-k4v", | |
| "ngram-mod", | |
| "ngram-cache" | |
| ]); | |
| function speculativeFlags(isSupported) { | |
| return Object.fromEntries( | |
| SPECULATIVE_DECODING_STRATEGIES.map((strategy) => [strategy, isSupported(strategy)]) | |
| ); | |
| } | |
| var NO_COMPLETION_CAPABILITIES = Object.freeze({ | |
| presencePenalty: false, | |
| minP: false, | |
| thinkingBudget: false, | |
| speculativeDecoding: Object.freeze(speculativeFlags(() => false)) | |
| }); | |
| var MAX_NGRAM_SIZE = 65535; | |
| var MAX_INT32 = 2147483647; | |
| var MAX_THINKING_BUDGET_TOKENS = 2147483647; | |
| function optionalNumber(value, name, isValid, range) { | |
| if (value == null) { | |
| return null; | |
| } | |
| if (typeof value !== "number" || !Number.isFinite(Math.fround(value)) || !isValid(value)) { | |
| throw new RangeError(`CompletionOptions.${name} must be a finite number${range}; got ${String(value)}.`); | |
| } | |
| return value; | |
| } | |
| function tag(budget, name) { | |
| const value = budget[name]; | |
| if (typeof value !== "string" || value.trim().length === 0) { | |
| throw new TypeError(`CompletionOptions.thinkingBudget.${name} must be a non-empty string.`); | |
| } | |
| return value; | |
| } | |
| function thinkingBudget(value, hasMediaParts) { | |
| if (value == null) { | |
| return null; | |
| } | |
| if (typeof value !== "object") { | |
| throw new TypeError("CompletionOptions.thinkingBudget must be an object."); | |
| } | |
| const budget = value; | |
| const maxTokens = budget.maxTokens; | |
| if (!Number.isInteger(maxTokens) || maxTokens < 0 || maxTokens > MAX_THINKING_BUDGET_TOKENS) { | |
| throw new RangeError( | |
| `CompletionOptions.thinkingBudget.maxTokens must be an integer from 0 to ${MAX_THINKING_BUDGET_TOKENS}; got ${String(maxTokens)}.` | |
| ); | |
| } | |
| const startTag = tag(budget, "startTag"); | |
| const endTag = tag(budget, "endTag"); | |
| const forcedMessage = budget.forcedMessage ?? ""; | |
| if (typeof forcedMessage !== "string") { | |
| throw new TypeError("CompletionOptions.thinkingBudget.forcedMessage must be a string."); | |
| } | |
| if (hasMediaParts) { | |
| throw new Error("CompletionOptions.thinkingBudget supports text-only prompts; remove the media parts."); | |
| } | |
| return { maxTokens, startTag, endTag, forcedMessage }; | |
| } | |
| function speculativeInteger(config, name, min, max) { | |
| const value = config[name]; | |
| if (value == null) { | |
| return null; | |
| } | |
| if (!Number.isInteger(value) || value < min || value > max) { | |
| throw new RangeError( | |
| `CompletionOptions.speculativeDecoding.${name} must be an integer from ${min} to ${max}; got ${String(value)}.` | |
| ); | |
| } | |
| return value; | |
| } | |
| function speculativeProbability(config, name) { | |
| const value = config[name]; | |
| if (value == null) { | |
| return null; | |
| } | |
| if (typeof value !== "number" || !(value >= 0 && value <= 1)) { | |
| throw new RangeError( | |
| `CompletionOptions.speculativeDecoding.${name} must be a number from 0 to 1; got ${String(value)}.` | |
| ); | |
| } | |
| return value; | |
| } | |
| function ngramCacheSource(config, name) { | |
| const value = config[name]; | |
| if (value == null) { | |
| return null; | |
| } | |
| const isBytes = value instanceof ArrayBuffer || ArrayBuffer.isView(value); | |
| if (!(typeof value === "string" || isBytes)) { | |
| throw new TypeError( | |
| `CompletionOptions.speculativeDecoding.${name} must be a URL string, an ArrayBuffer or a typed array.` | |
| ); | |
| } | |
| if ((typeof value === "string" ? value.length : value.byteLength) === 0) { | |
| throw new TypeError(`CompletionOptions.speculativeDecoding.${name} is empty.`); | |
| } | |
| return value; | |
| } | |
| function isDraftStrategy(strategy) { | |
| return strategy.startsWith("draft-"); | |
| } | |
| function speculativeDecoding(value, conflicts) { | |
| if (value == null) { | |
| return null; | |
| } | |
| if (typeof value !== "object") { | |
| throw new TypeError("CompletionOptions.speculativeDecoding must be an object."); | |
| } | |
| const config = value; | |
| const requested = config.strategies; | |
| if (!Array.isArray(requested) || requested.length === 0) { | |
| throw new TypeError("CompletionOptions.speculativeDecoding.strategies must be a non-empty array."); | |
| } | |
| const strategies = []; | |
| for (const strategy of requested) { | |
| if (!SPECULATIVE_DECODING_STRATEGIES.includes(strategy)) { | |
| throw new TypeError( | |
| `CompletionOptions.speculativeDecoding.strategies has an unknown strategy ${JSON.stringify(strategy)}; expected one of ${SPECULATIVE_DECODING_STRATEGIES.join(", ")}.` | |
| ); | |
| } | |
| if (!strategies.includes(strategy)) { | |
| strategies.push(strategy); | |
| } | |
| } | |
| const draftTokenMax = speculativeInteger(config, "draftTokenMax", 0, MAX_INT32); | |
| const draftTokenMin = speculativeInteger(config, "draftTokenMin", 0, MAX_INT32); | |
| const minProbability = speculativeProbability(config, "minProbability"); | |
| const draftSplitProbability = speculativeProbability(config, "draftSplitProbability"); | |
| const ngramSizeN = speculativeInteger(config, "ngramSizeN", 1, MAX_NGRAM_SIZE); | |
| const ngramSizeM = speculativeInteger(config, "ngramSizeM", 1, MAX_NGRAM_SIZE); | |
| const ngramMinHits = speculativeInteger(config, "ngramMinHits", 1, MAX_NGRAM_SIZE); | |
| const ngramMatch = speculativeInteger(config, "ngramMatch", 1, MAX_NGRAM_SIZE); | |
| const ngramTokenMin = speculativeInteger(config, "ngramTokenMin", 0, MAX_INT32); | |
| const ngramTokenMax = speculativeInteger(config, "ngramTokenMax", 0, MAX_INT32); | |
| const ngramCacheStatic = ngramCacheSource(config, "ngramCacheStatic"); | |
| const ngramCacheDynamic = ngramCacheSource(config, "ngramCacheDynamic"); | |
| if (strategies.filter(isDraftStrategy).length > 1) { | |
| throw new Error( | |
| "CompletionOptions.speculativeDecoding can mix n-gram strategies with at most one draft-* strategy." | |
| ); | |
| } | |
| if (!strategies.some(isDraftStrategy) && (draftTokenMin != null || minProbability != null || draftSplitProbability != null)) { | |
| throw new Error( | |
| "CompletionOptions.speculativeDecoding: n-gram strategies use token history and do not support draftTokenMin, minProbability or draftSplitProbability unless a draft-* strategy is also enabled." | |
| ); | |
| } | |
| if ((ngramCacheStatic != null || ngramCacheDynamic != null) && !strategies.includes("ngram-cache")) { | |
| throw new Error( | |
| "CompletionOptions.speculativeDecoding: ngramCacheStatic and ngramCacheDynamic need the ngram-cache strategy." | |
| ); | |
| } | |
| if (ngramTokenMin != null && ngramTokenMax != null && ngramTokenMin > ngramTokenMax) { | |
| throw new RangeError( | |
| `CompletionOptions.speculativeDecoding.ngramTokenMin (${ngramTokenMin}) must not exceed ngramTokenMax (${ngramTokenMax}).` | |
| ); | |
| } | |
| let resolvedDraftTokenMax = 0; | |
| for (const strategy of strategies) { | |
| let strategyMax; | |
| if (isDraftStrategy(strategy)) { | |
| strategyMax = draftTokenMax ?? 3; | |
| } else if (strategy === "ngram-mod") { | |
| strategyMax = ngramTokenMax ?? draftTokenMax ?? 64; | |
| } else if (strategy === "ngram-cache") { | |
| strategyMax = draftTokenMax ?? 8; | |
| } else { | |
| strategyMax = ngramSizeM ?? 48; | |
| } | |
| resolvedDraftTokenMax = Math.max(resolvedDraftTokenMax, strategyMax); | |
| } | |
| if (resolvedDraftTokenMax === 0) { | |
| resolvedDraftTokenMax = 64; | |
| } | |
| if ((draftTokenMin ?? 0) > resolvedDraftTokenMax) { | |
| throw new RangeError( | |
| `CompletionOptions.speculativeDecoding.draftTokenMin (${draftTokenMin}) must not exceed the draft token maximum (${resolvedDraftTokenMax}).` | |
| ); | |
| } | |
| if (conflicts.hasMediaParts) { | |
| throw new Error("CompletionOptions.speculativeDecoding supports text-only prompts; remove the media parts."); | |
| } | |
| if (conflicts.hasThinkingBudget) { | |
| throw new Error("CompletionOptions.speculativeDecoding cannot be combined with CompletionOptions.thinkingBudget."); | |
| } | |
| if (conflicts.hasGrammar) { | |
| throw new Error("CompletionOptions.speculativeDecoding does not support CompletionOptions.grammar."); | |
| } | |
| return { | |
| strategies, | |
| draftTokenMax: resolvedDraftTokenMax, | |
| draftTokenMin: draftTokenMin ?? 0, | |
| minProbability: minProbability ?? -1, | |
| draftSplitProbability: draftSplitProbability ?? -1, | |
| ngramSizeN: ngramSizeN ?? 0, | |
| ngramSizeM: ngramSizeM ?? 0, | |
| ngramMinHits: ngramMinHits ?? 0, | |
| ngramMatch: ngramMatch ?? 0, | |
| ngramTokenMin: ngramTokenMin ?? -1, | |
| ngramTokenMax: ngramTokenMax ?? (strategies.includes("ngram-mod") ? draftTokenMax ?? 0 : 0), | |
| ngramCacheStatic, | |
| ngramCacheDynamic | |
| }; | |
| } | |
| function resolveCompletionSamplingOptions(options) { | |
| const hasMediaParts = Array.isArray(options?.parts) && options.parts.length > 0; | |
| const budget = thinkingBudget(options?.thinkingBudget, hasMediaParts); | |
| return { | |
| minP: optionalNumber(options?.minP, "minP", (value) => value >= 0 && value <= 1, " from 0 to 1") ?? 0, | |
| presencePenalty: optionalNumber(options?.presencePenalty, "presencePenalty", () => true, "") ?? 0, | |
| thinkingBudget: budget, | |
| speculative: speculativeDecoding(options?.speculativeDecoding, { | |
| hasMediaParts, | |
| hasGrammar: typeof options?.grammar === "string" && options.grammar.length > 0, | |
| hasThinkingBudget: budget != null | |
| }) | |
| }; | |
| } | |
| function completionCapabilitiesFrom(raw) { | |
| let parsed = null; | |
| try { | |
| parsed = JSON.parse(raw || "{}"); | |
| } catch (_) { | |
| return { ...NO_COMPLETION_CAPABILITIES }; | |
| } | |
| const speculative = parsed?.speculativeDecoding; | |
| return { | |
| presencePenalty: parsed?.presencePenalty === true, | |
| minP: parsed?.minP === true, | |
| thinkingBudget: parsed?.thinkingBudget === true, | |
| speculativeDecoding: speculativeFlags((strategy) => speculative?.[strategy] === true) | |
| }; | |
| } | |
| function requireCompletionCapabilities(sampling, capabilities) { | |
| const missing = [ | |
| sampling.minP !== 0 && !capabilities.minP ? "minP" : null, | |
| sampling.presencePenalty !== 0 && !capabilities.presencePenalty ? "presencePenalty" : null, | |
| sampling.thinkingBudget != null && !capabilities.thinkingBudget ? "thinkingBudget" : null, | |
| sampling.speculative != null && !Object.values(capabilities.speculativeDecoding).includes(true) ? "speculativeDecoding" : null | |
| ].filter((name) => name != null); | |
| if (missing.length > 0) { | |
| throw new Error( | |
| `The loaded WebGPU core does not support CompletionOptions.${missing.join(" or CompletionOptions.")}.` | |
| ); | |
| } | |
| } | |
| // js/src/internal/constants.ts | |
| var defaultModelCacheName = "llamadart-webgpu-model-cache-v1"; | |
| var BRIDGE_DISPOSED_MESSAGE = "Bridge has been disposed."; | |
| var GENERATION_ALREADY_ACTIVE_RC = -7; | |
| var GENERATION_ALREADY_ACTIVE_MESSAGE = "Generation is already active on this bridge runtime."; | |
| var INVALID_GRAMMAR_ERROR_TEXT = "Failed to initialize sampler chain (invalid grammar)"; | |
| // js/src/internal/typed_values.ts | |
| function toUint8Array(value) { | |
| if (!value) { | |
| return null; | |
| } | |
| if (value instanceof Uint8Array) { | |
| return value; | |
| } | |
| if (ArrayBuffer.isView(value)) { | |
| return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); | |
| } | |
| if (value instanceof ArrayBuffer) { | |
| return new Uint8Array(value); | |
| } | |
| if (Array.isArray(value)) { | |
| return Uint8Array.from(value.map((v) => Number(v) & 255)); | |
| } | |
| return null; | |
| } | |
| function bufferSourceBytes(source) { | |
| if (source instanceof ArrayBuffer) { | |
| return new Uint8Array(source); | |
| } | |
| if (ArrayBuffer.isView(source)) { | |
| return new Uint8Array(source.buffer, source.byteOffset, source.byteLength); | |
| } | |
| return null; | |
| } | |
| function toFloat32Array(value) { | |
| if (!value) { | |
| return null; | |
| } | |
| if (value instanceof Float32Array) { | |
| return value; | |
| } | |
| if (ArrayBuffer.isView(value)) { | |
| return new Float32Array( | |
| value.buffer, | |
| value.byteOffset, | |
| Math.floor(value.byteLength / Float32Array.BYTES_PER_ELEMENT) | |
| ); | |
| } | |
| if (value instanceof ArrayBuffer) { | |
| return new Float32Array(value); | |
| } | |
| if (Array.isArray(value)) { | |
| return Float32Array.from(value.map((v) => Number(v) || 0)); | |
| } | |
| return null; | |
| } | |
| function isInt32(value) { | |
| return typeof value === "number" && Number.isInteger(value) && value >= -2147483648 && value <= 2147483647; | |
| } | |
| // js/src/internal/decision.ts | |
| var DECISION_API_VERSION = 1; | |
| var DECISION_WORKER_TIMEOUT_PER_SEQUENCE_MS = 60 * 1e3; | |
| function decisionHandleFrom(handle) { | |
| if (!isInt32(handle) || handle <= 0) { | |
| throw new TypeError(`Decision head handle must be a positive integer, got ${String(handle)}.`); | |
| } | |
| return handle; | |
| } | |
| function decisionIntegerList(value, label) { | |
| const isList = Array.isArray(value) || ArrayBuffer.isView(value) && !(value instanceof DataView); | |
| if (!isList) { | |
| throw new TypeError(`${label} must be an array or typed array of integers.`); | |
| } | |
| const list = value; | |
| const out = new Int32Array(list.length); | |
| for (let index = 0; index < list.length; index += 1) { | |
| const item = list[index]; | |
| if (!isInt32(item)) { | |
| throw new TypeError(`${label}[${index}] is ${String(item)}; expected a 32-bit integer.`); | |
| } | |
| out[index] = item; | |
| } | |
| return out; | |
| } | |
| function normalizeDecisionSequences(sequences) { | |
| if (!Array.isArray(sequences)) { | |
| throw new TypeError("Decision sequences must be an array."); | |
| } | |
| return sequences.map((sequence, index) => { | |
| const label = `Decision sequence ${index}`; | |
| if (!sequence || typeof sequence !== "object") { | |
| throw new TypeError(`${label} must be an object with tokens, markers and questionType.`); | |
| } | |
| const tokens = decisionIntegerList(sequence.tokens, `${label} tokens`); | |
| const markers = decisionIntegerList(sequence.markers, `${label} markers`); | |
| const questionType = sequence.questionType; | |
| if (!isInt32(questionType)) { | |
| throw new TypeError( | |
| `${label} questionType is ${String(questionType)}; expected 0 (choice), 1 (score) or 2 (noul).` | |
| ); | |
| } | |
| return { tokens, markers, questionType }; | |
| }); | |
| } | |
| function encodeDecisionSequences(sequences) { | |
| const encoded = normalizeDecisionSequences(sequences); | |
| let words = 1; | |
| for (const sequence of encoded) { | |
| words += 3 + sequence.tokens.length + sequence.markers.length; | |
| } | |
| const bytes = new Uint8Array(words * 4); | |
| const view = new DataView(bytes.buffer); | |
| let offset = 0; | |
| const put = (value) => { | |
| view.setInt32(offset, value, true); | |
| offset += 4; | |
| }; | |
| put(encoded.length); | |
| for (const sequence of encoded) { | |
| put(sequence.questionType); | |
| put(sequence.tokens.length); | |
| put(sequence.markers.length); | |
| sequence.tokens.forEach(put); | |
| sequence.markers.forEach(put); | |
| } | |
| return bytes; | |
| } | |
| function decodeDecisionOutputs(bytes, count) { | |
| const malformed = () => new Error("Decision output from the WebGPU core is malformed."); | |
| const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); | |
| let offset = 0; | |
| const floats = (length) => { | |
| const values = new Float32Array(length); | |
| for (let index = 0; index < length; index += 1) { | |
| values[index] = view.getFloat32(offset, true); | |
| offset += 4; | |
| } | |
| return values; | |
| }; | |
| const outputs = []; | |
| for (let index = 0; index < count; index += 1) { | |
| if (bytes.byteLength - offset < 8) { | |
| throw malformed(); | |
| } | |
| const logitCount = view.getInt32(offset, true); | |
| const actCount = view.getInt32(offset + 4, true); | |
| offset += 8; | |
| if (logitCount < 0 || actCount < 0 || (logitCount + actCount) * 4 > bytes.byteLength - offset) { | |
| throw malformed(); | |
| } | |
| outputs.push({ logits: floats(logitCount), actLogits: floats(actCount) }); | |
| } | |
| if (offset !== bytes.byteLength) { | |
| throw malformed(); | |
| } | |
| return outputs; | |
| } | |
| // js/src/internal/logging.ts | |
| function logLevelForName(level) { | |
| switch (level) { | |
| case "debug": | |
| return 0; | |
| case "log": | |
| case "info": | |
| return 1; | |
| case "warn": | |
| return 2; | |
| case "error": | |
| return 3; | |
| default: | |
| return 1; | |
| } | |
| } | |
| function logThresholdForConfiguredLevel(level) { | |
| switch (level) { | |
| case 0: | |
| return 99; | |
| case 1: | |
| return 0; | |
| case 2: | |
| return 1; | |
| case 3: | |
| return 2; | |
| case 4: | |
| return 3; | |
| default: | |
| return 1; | |
| } | |
| } | |
| // js/src/internal/lora.ts | |
| var LORA_API_VERSION = 1; | |
| var LORA_LOAD_ABORT_MESSAGE = "LoRA adapter load was cancelled."; | |
| function loraHandleFrom(handle) { | |
| if (!isInt32(handle) || handle <= 0) { | |
| throw new TypeError(`LoRA adapter handle must be a positive integer, got ${String(handle)}.`); | |
| } | |
| return handle; | |
| } | |
| function loraScaleFrom(scale) { | |
| if (typeof scale !== "number" || !Number.isFinite(scale)) { | |
| throw new TypeError(`LoRA adapter scale must be a finite number, got ${String(scale)}.`); | |
| } | |
| if (!Number.isFinite(Math.fround(scale))) { | |
| throw new RangeError(`LoRA adapter scale ${String(scale)} is outside the 32-bit float range.`); | |
| } | |
| return scale; | |
| } | |
| function staleLoraAdapterError(handle) { | |
| return new Error( | |
| `LoRA adapter ${handle} is not loaded; its model was unloaded or replaced. Load the adapter again.` | |
| ); | |
| } | |
| // js/src/internal/model_source.ts | |
| function cloneModelSource(source) { | |
| if (Array.isArray(source)) { | |
| return source.slice(); | |
| } | |
| return String(source || "").trim(); | |
| } | |
| function hasModelSource(source) { | |
| return Array.isArray(source) ? source.length > 0 : typeof source === "string" && source.length > 0; | |
| } | |
| function basenameFromUrl(url) { | |
| try { | |
| const parsed = new URL(url, typeof window !== "undefined" ? window.location.href : void 0); | |
| const pathname = parsed.pathname || ""; | |
| const name = pathname.split("/").pop() || "model.gguf"; | |
| return name.includes("?") ? name.split("?")[0] : name; | |
| } catch (_) { | |
| const parts = String(url).split("/"); | |
| return parts[parts.length - 1] || "model.gguf"; | |
| } | |
| } | |
| function normalizeAbsoluteUrl(url) { | |
| try { | |
| return new URL(url, typeof window !== "undefined" ? window.location.href : void 0).toString(); | |
| } catch (_) { | |
| return String(url); | |
| } | |
| } | |
| function parseSplitShardPattern(fileName) { | |
| if (typeof fileName !== "string" || fileName.length === 0) { | |
| return null; | |
| } | |
| const match = fileName.match(/^(.*)-(\d{4,6})-of-(\d{4,6})\.gguf$/i); | |
| if (!match) { | |
| return null; | |
| } | |
| const total = Number(match[3]); | |
| if (!Number.isInteger(total) || total < 2 || total > 512) { | |
| return null; | |
| } | |
| return { | |
| prefix: match[1], | |
| width: Math.max(match[2].length, match[3].length), | |
| total | |
| }; | |
| } | |
| function expandModelShardUrls(modelUrlOrUrls) { | |
| if (Array.isArray(modelUrlOrUrls)) { | |
| return modelUrlOrUrls.map((value) => String(value || "").trim()).filter((value) => value.length > 0).map((value) => normalizeAbsoluteUrl(value)); | |
| } | |
| const source = String(modelUrlOrUrls || "").trim(); | |
| if (source.length === 0) { | |
| return []; | |
| } | |
| try { | |
| const parsed = new URL(source, typeof window !== "undefined" ? window.location.href : void 0); | |
| const pathname = parsed.pathname || ""; | |
| const slash = pathname.lastIndexOf("/"); | |
| const dirPath = slash >= 0 ? pathname.slice(0, slash + 1) : ""; | |
| const fileName = slash >= 0 ? pathname.slice(slash + 1) : pathname; | |
| const split = parseSplitShardPattern(fileName); | |
| if (!split) { | |
| return [parsed.toString()]; | |
| } | |
| const totalShardId = String(split.total).padStart(split.width, "0"); | |
| const urls = []; | |
| for (let shardIndex = 1; shardIndex <= split.total; shardIndex += 1) { | |
| const shardId = String(shardIndex).padStart(split.width, "0"); | |
| parsed.pathname = `${dirPath}${split.prefix}-${shardId}-of-${totalShardId}.gguf`; | |
| urls.push(parsed.toString()); | |
| } | |
| return urls; | |
| } catch (_) { | |
| return [source]; | |
| } | |
| } | |
| // js/src/internal/core_loader.ts | |
| function normalizeFactory(moduleExport) { | |
| if (typeof moduleExport === "function") { | |
| return moduleExport; | |
| } | |
| if (moduleExport && typeof moduleExport.default === "function") { | |
| return moduleExport.default; | |
| } | |
| if (moduleExport && typeof moduleExport.createLlamaWebGpuCoreModule === "function") { | |
| return moduleExport.createLlamaWebGpuCoreModule; | |
| } | |
| throw new Error("Unable to resolve llama_webgpu_core factory function"); | |
| } | |
| async function importCoreFactory(moduleUrl) { | |
| const exportedModule = await import(moduleUrl); | |
| return normalizeFactory(exportedModule); | |
| } | |
| // js/src/internal/parse.ts | |
| function parsePositiveInteger(value) { | |
| const numeric = Number(value); | |
| if (!Number.isFinite(numeric) || numeric <= 0) { | |
| return 0; | |
| } | |
| return Math.trunc(numeric); | |
| } | |
| function parseInteger(value, fallback = 0) { | |
| const numeric = Number(value); | |
| if (!Number.isFinite(numeric)) { | |
| return fallback; | |
| } | |
| return Math.trunc(numeric); | |
| } | |
| function parseBooleanFlag(value, fallback = false) { | |
| if (typeof value === "boolean") { | |
| return value; | |
| } | |
| if (typeof value === "number" && Number.isFinite(value)) { | |
| return value !== 0; | |
| } | |
| return fallback; | |
| } | |
| function parseOptionalBooleanFlag(value) { | |
| if (typeof value === "boolean") { | |
| return value ? 1 : 0; | |
| } | |
| if (typeof value === "number" && Number.isFinite(value)) { | |
| return value !== 0 ? 1 : 0; | |
| } | |
| return -1; | |
| } | |
| function parseEnumValue(value, allowed, fallback) { | |
| const parsed = parseInteger(value, fallback); | |
| return allowed.includes(parsed) ? parsed : fallback; | |
| } | |
| function parsePositiveNumber(value) { | |
| const numeric = Number(value); | |
| return Number.isFinite(numeric) && numeric > 0 ? numeric : 0; | |
| } | |
| // js/src/internal/download.ts | |
| function hasReadableResponseStream(response) { | |
| return !!(response && response.body && typeof response.body.getReader === "function"); | |
| } | |
| function sumProgressValues(values) { | |
| let total = 0; | |
| for (const value of values || []) { | |
| const numeric = Number(value); | |
| if (Number.isFinite(numeric) && numeric > 0) { | |
| total += numeric; | |
| } | |
| } | |
| return total; | |
| } | |
| function parseTotalFromContentRangeHeader(contentRangeHeader) { | |
| if (typeof contentRangeHeader !== "string" || contentRangeHeader.length === 0) { | |
| return 0; | |
| } | |
| const slash = contentRangeHeader.lastIndexOf("/"); | |
| if (slash < 0 || slash + 1 >= contentRangeHeader.length) { | |
| return 0; | |
| } | |
| return parsePositiveInteger(contentRangeHeader.slice(slash + 1)); | |
| } | |
| function inferResponseTotalBytes(response, loadedFallback = 0) { | |
| if (!response || !response.headers) { | |
| return parsePositiveInteger(loadedFallback); | |
| } | |
| const linkedSize = parsePositiveInteger(response.headers.get("x-linked-size")); | |
| if (linkedSize > 0) { | |
| return linkedSize; | |
| } | |
| const contentRangeTotal = parseTotalFromContentRangeHeader( | |
| response.headers.get("content-range") | |
| ); | |
| if (contentRangeTotal > 0) { | |
| return contentRangeTotal; | |
| } | |
| const contentLength = parsePositiveInteger(response.headers.get("content-length")); | |
| if (contentLength > 0) { | |
| return contentLength; | |
| } | |
| return parsePositiveInteger(loadedFallback); | |
| } | |
| function isRetryableStreamNetworkError(error) { | |
| const text = String(error || "").toLowerCase(); | |
| return text.includes("network error") || text.includes("failed to fetch") || text.includes("networkerror") || text.includes("err_network_io_suspended") || text.includes("the network connection was lost") || text.includes("connection reset") || text.includes("timeout") || text.includes("timed out"); | |
| } | |
| async function readStreamChunkWithTimeout(reader, timeoutMs, label = "stream read") { | |
| const resolvedTimeout = Number(timeoutMs); | |
| if (!Number.isFinite(resolvedTimeout) || resolvedTimeout <= 0) { | |
| return reader.read(); | |
| } | |
| let timeoutHandle = null; | |
| try { | |
| return await Promise.race([ | |
| reader.read(), | |
| new Promise((_, reject) => { | |
| timeoutHandle = globalThis.setTimeout(() => { | |
| reject(new Error(`${label} timeout (${resolvedTimeout}ms)`)); | |
| }, resolvedTimeout); | |
| }) | |
| ]); | |
| } finally { | |
| if (timeoutHandle != null) { | |
| globalThis.clearTimeout(timeoutHandle); | |
| } | |
| } | |
| } | |
| async function drainResponseWithProgress(response, progressCallback, options = {}) { | |
| const total = Number(response.headers.get("content-length")) || 0; | |
| const chunkTimeoutMs = parsePositiveInteger(options.chunkTimeoutMs); | |
| if (!response.body || typeof response.body.getReader !== "function") { | |
| const bytes = new Uint8Array(await response.arrayBuffer()); | |
| if (typeof progressCallback === "function") { | |
| progressCallback({ loaded: bytes.byteLength, total: total || bytes.byteLength }); | |
| } | |
| return bytes.byteLength; | |
| } | |
| const reader = response.body.getReader(); | |
| let loaded = 0; | |
| let lastBucket = -1; | |
| while (true) { | |
| const { done, value } = await readStreamChunkWithTimeout( | |
| reader, | |
| chunkTimeoutMs, | |
| "response drain read" | |
| ); | |
| if (done) { | |
| break; | |
| } | |
| if (!value || value.length === 0) { | |
| continue; | |
| } | |
| loaded += value.length; | |
| if (typeof progressCallback === "function") { | |
| const effectiveTotal = total || loaded; | |
| const bucket = effectiveTotal > 0 ? Math.floor(loaded / effectiveTotal * 100) : -1; | |
| if (bucket > lastBucket) { | |
| lastBucket = bucket; | |
| progressCallback({ loaded, total: effectiveTotal }); | |
| } | |
| } | |
| } | |
| if (typeof progressCallback === "function") { | |
| progressCallback({ loaded, total: total || loaded }); | |
| } | |
| return loaded; | |
| } | |
| function ensureFsDirectory(fs, dirPath) { | |
| const slash = dirPath.lastIndexOf("/"); | |
| const parent = slash > 0 ? dirPath.slice(0, slash) : "/"; | |
| const name = dirPath.slice(slash + 1); | |
| const exists = () => { | |
| try { | |
| const entries = fs.readdir(parent); | |
| return Array.isArray(entries) && entries.includes(name); | |
| } catch (_) { | |
| return false; | |
| } | |
| }; | |
| if (exists()) { | |
| return; | |
| } | |
| try { | |
| fs.mkdir(dirPath); | |
| } catch (error) { | |
| if (!exists()) { | |
| throw error; | |
| } | |
| } | |
| } | |
| function unlinkFsFile(fs, filePath) { | |
| try { | |
| fs.unlink(filePath); | |
| return true; | |
| } catch (_) { | |
| return false; | |
| } | |
| } | |
| async function writeResponseToFsFileWithProgress(response, fs, filePath, progressCallback, writeOptions = {}) { | |
| const total = parsePositiveInteger(writeOptions.totalBytes) || inferResponseTotalBytes(response, 0); | |
| const useBigIntPosition = writeOptions.useBigIntPosition === true; | |
| const startOffset = parsePositiveInteger(writeOptions.startOffset); | |
| const preservePartialOnError = writeOptions.preservePartialOnError === true; | |
| const appendMode = startOffset > 0; | |
| const chunkTimeoutMs = parsePositiveInteger(writeOptions.chunkTimeoutMs); | |
| const signal = writeOptions.signal || null; | |
| const abortMessage = writeOptions.abortMessage || "Model transfer was cancelled."; | |
| throwIfAborted(signal, abortMessage); | |
| if (!appendMode) { | |
| unlinkFsFile(fs, filePath); | |
| } | |
| if (!response.body || typeof response.body.getReader !== "function") { | |
| throwIfAborted(signal, abortMessage); | |
| const bytes = new Uint8Array(await response.arrayBuffer()); | |
| throwIfAborted(signal, abortMessage); | |
| if (appendMode) { | |
| const stream2 = fs.open(filePath, "r+"); | |
| try { | |
| throwIfAborted(signal, abortMessage); | |
| const position = useBigIntPosition ? BigInt(startOffset) : startOffset; | |
| fs.write(stream2, bytes, 0, bytes.length, position); | |
| } finally { | |
| fs.close(stream2); | |
| } | |
| } else { | |
| fs.writeFile(filePath, bytes); | |
| } | |
| const finalLoaded2 = startOffset + bytes.byteLength; | |
| if (typeof progressCallback === "function") { | |
| progressCallback({ loaded: finalLoaded2, total: total || finalLoaded2 }); | |
| } | |
| return finalLoaded2; | |
| } | |
| const reader = response.body.getReader(); | |
| const openMode = appendMode ? "r+" : "w"; | |
| const stream = fs.open(filePath, openMode); | |
| let loaded = 0; | |
| let lastBucket = -1; | |
| let writePosition = startOffset; | |
| let writePositionBigInt = null; | |
| try { | |
| while (true) { | |
| throwIfAborted(signal, abortMessage); | |
| const { done, value } = await readStreamChunkWithTimeout( | |
| reader, | |
| chunkTimeoutMs, | |
| "response file read" | |
| ); | |
| if (done) { | |
| throwIfAborted(signal, abortMessage); | |
| break; | |
| } | |
| if (!value || value.length === 0) { | |
| continue; | |
| } | |
| const chunk = value.slice ? value.slice() : new Uint8Array(value); | |
| throwIfAborted(signal, abortMessage); | |
| if (useBigIntPosition) { | |
| if (writePositionBigInt == null) { | |
| writePositionBigInt = BigInt(startOffset); | |
| } | |
| fs.write(stream, chunk, 0, chunk.length, writePositionBigInt); | |
| writePositionBigInt += BigInt(chunk.length); | |
| } else { | |
| fs.write(stream, chunk, 0, chunk.length, writePosition); | |
| writePosition += chunk.length; | |
| } | |
| loaded += chunk.length; | |
| if (typeof progressCallback === "function") { | |
| const effectiveLoaded = startOffset + loaded; | |
| const effectiveTotal = total || effectiveLoaded; | |
| const bucket = effectiveTotal > 0 ? Math.floor(effectiveLoaded / effectiveTotal * 100) : -1; | |
| if (bucket > lastBucket) { | |
| lastBucket = bucket; | |
| progressCallback({ loaded: effectiveLoaded, total: effectiveTotal }); | |
| } | |
| } | |
| } | |
| } catch (error) { | |
| if (signal?.aborted && error?.name !== "AbortError") { | |
| error = createAbortError(abortMessage); | |
| } | |
| try { | |
| if (error && typeof error === "object") { | |
| error.llamadartLoadedBytes = startOffset + loaded; | |
| error.llamadartFilePath = filePath; | |
| } | |
| } catch (_) { | |
| } | |
| try { | |
| fs.close(stream); | |
| } catch (_) { | |
| } | |
| try { | |
| await reader.cancel?.(); | |
| } catch (_) { | |
| } | |
| if (!preservePartialOnError || signal?.aborted || error?.name === "AbortError") { | |
| try { | |
| fs.unlink(filePath); | |
| } catch (_) { | |
| } | |
| } | |
| throw error; | |
| } | |
| fs.close(stream); | |
| const finalLoaded = startOffset + loaded; | |
| if (typeof progressCallback === "function") { | |
| progressCallback({ loaded: finalLoaded, total: total || finalLoaded }); | |
| } | |
| return finalLoaded; | |
| } | |
| // js/src/internal/environment.ts | |
| function isSafariUserAgent(userAgent) { | |
| if (typeof userAgent !== "string" || userAgent.length === 0) { | |
| return false; | |
| } | |
| const hasSafariToken = /Safari\//.test(userAgent); | |
| const hasOtherBrowserToken = /(Chrome|Chromium|CriOS|Edg|OPR|Firefox|FxiOS)\//.test(userAgent); | |
| return hasSafariToken && !hasOtherBrowserToken; | |
| } | |
| function isCrossOriginIsolatedRuntime() { | |
| try { | |
| if (typeof globalThis.crossOriginIsolated === "boolean") { | |
| return globalThis.crossOriginIsolated; | |
| } | |
| } catch (_) { | |
| } | |
| return true; | |
| } | |
| // js/src/internal/image.ts | |
| async function decodeImageBytesToRgb(bytes, options = {}) { | |
| const sourceBytes = toUint8Array(bytes); | |
| if (!sourceBytes || sourceBytes.length === 0) { | |
| return null; | |
| } | |
| if (typeof createImageBitmap !== "function" || typeof Blob !== "function") { | |
| return null; | |
| } | |
| const maxPixelsCandidate = Number(options.maxPixels); | |
| const maxPixels = Number.isFinite(maxPixelsCandidate) && maxPixelsCandidate > 0 ? Math.max(65536, Math.min(33554432, Math.trunc(maxPixelsCandidate))) : 0; | |
| const maxEdgeCandidate = Number(options.maxEdge); | |
| const maxEdge = Number.isFinite(maxEdgeCandidate) && maxEdgeCandidate > 0 ? Math.max(64, Math.min(16384, Math.trunc(maxEdgeCandidate))) : 0; | |
| if (maxPixels <= 0 && maxEdge <= 0) { | |
| return null; | |
| } | |
| let bitmap = null; | |
| try { | |
| const mimeType = typeof options.mimeType === "string" && options.mimeType.length > 0 ? options.mimeType : "image/png"; | |
| const blob = new Blob([sourceBytes], { type: mimeType }); | |
| bitmap = await createImageBitmap(blob); | |
| const sourceWidth = Math.max(1, Math.trunc(Number(bitmap.width) || 0)); | |
| const sourceHeight = Math.max(1, Math.trunc(Number(bitmap.height) || 0)); | |
| let scale = 1; | |
| if (maxPixels > 0) { | |
| const sourcePixels = sourceWidth * sourceHeight; | |
| if (sourcePixels > maxPixels) { | |
| scale = Math.min(scale, Math.sqrt(maxPixels / sourcePixels)); | |
| } | |
| } | |
| if (maxEdge > 0) { | |
| const sourceLongest = Math.max(sourceWidth, sourceHeight); | |
| if (sourceLongest > maxEdge) { | |
| scale = Math.min(scale, maxEdge / sourceLongest); | |
| } | |
| } | |
| const width = Math.max(1, Math.round(sourceWidth * scale)); | |
| const height = Math.max(1, Math.round(sourceHeight * scale)); | |
| let canvas = null; | |
| let context = null; | |
| if (typeof OffscreenCanvas === "function") { | |
| canvas = new OffscreenCanvas(width, height); | |
| context = canvas.getContext("2d", { | |
| alpha: false, | |
| willReadFrequently: true | |
| }); | |
| } | |
| if (!context && typeof document !== "undefined" && typeof document.createElement === "function") { | |
| canvas = document.createElement("canvas"); | |
| canvas.width = width; | |
| canvas.height = height; | |
| context = canvas.getContext("2d", { | |
| alpha: false, | |
| willReadFrequently: true | |
| }); | |
| } | |
| if (!context) { | |
| return null; | |
| } | |
| context.drawImage(bitmap, 0, 0, width, height); | |
| let encodedBytes = null; | |
| if (canvas && "convertToBlob" in canvas && typeof canvas.convertToBlob === "function") { | |
| const encodedBlob = await canvas.convertToBlob({ type: "image/png" }); | |
| if (encodedBlob) { | |
| encodedBytes = new Uint8Array(await encodedBlob.arrayBuffer()); | |
| } | |
| } | |
| if (!encodedBytes && canvas && "toBlob" in canvas && typeof canvas.toBlob === "function" && typeof Promise === "function") { | |
| const encodedBlob = await new Promise((resolve) => { | |
| canvas.toBlob((value) => { | |
| resolve(value || null); | |
| }, "image/png"); | |
| }); | |
| if (encodedBlob) { | |
| encodedBytes = new Uint8Array(await encodedBlob.arrayBuffer()); | |
| } | |
| } | |
| if (!encodedBytes || encodedBytes.length === 0) { | |
| return null; | |
| } | |
| return { | |
| bytes: encodedBytes, | |
| width, | |
| height, | |
| sourceWidth, | |
| sourceHeight, | |
| resized: width !== sourceWidth || height !== sourceHeight | |
| }; | |
| } catch (_) { | |
| return null; | |
| } finally { | |
| try { | |
| bitmap?.close?.(); | |
| } catch (_) { | |
| } | |
| } | |
| } | |
| // js/src/internal/next_token_scores.ts | |
| function nextTokenCandidateIds(value) { | |
| if (value == null) { | |
| return []; | |
| } | |
| const isList = Array.isArray(value) || ArrayBuffer.isView(value) && !(value instanceof DataView); | |
| if (!isList) { | |
| throw new TypeError("Next-token candidates must be an array or typed array of token ids."); | |
| } | |
| return Array.from(value, (item, index) => { | |
| if (!isInt32(item)) { | |
| throw new TypeError(`Next-token candidates[${index}] is ${String(item)}; expected a 32-bit integer.`); | |
| } | |
| return item; | |
| }); | |
| } | |
| function scoredTokensFrom(entries) { | |
| if (!Array.isArray(entries)) { | |
| return []; | |
| } | |
| return entries.map((entry) => ({ | |
| token: Number(entry?.token), | |
| bytes: Uint8Array.from(Array.isArray(entry?.bytes) ? entry.bytes : []), | |
| logprob: typeof entry?.logprob === "number" ? entry.logprob : -Infinity | |
| })); | |
| } | |
| // js/src/internal/text.ts | |
| function buildPromptFromMessages(messages, addAssistant) { | |
| const lines = []; | |
| for (const msg of messages || []) { | |
| const role = String(msg?.role ?? "user"); | |
| const content = String(msg?.content ?? ""); | |
| lines.push(`${role}: ${content}`); | |
| } | |
| if (addAssistant) { | |
| lines.push("assistant: "); | |
| } | |
| return lines.join("\n"); | |
| } | |
| function looksLikeCorruptedGeneration(text) { | |
| if (typeof text !== "string" || text.length === 0) { | |
| return false; | |
| } | |
| const normalized = text.trim(); | |
| if (normalized.length === 0) { | |
| return false; | |
| } | |
| const unusedTokens = text.match(/<unused\d+>/g) || []; | |
| if (unusedTokens.length >= 4) { | |
| return true; | |
| } | |
| const tokenLikeTags = text.match(/<[^>]{1,40}>/g) || []; | |
| if (tokenLikeTags.length >= 8) { | |
| return true; | |
| } | |
| const compact = text.replace(/\s+/g, ""); | |
| if (compact.length === 0) { | |
| return false; | |
| } | |
| const tagRun = compact.match(/(?:<[^>]{2,32}>){6,}/); | |
| if (tagRun) { | |
| return true; | |
| } | |
| const alphaNum = (normalized.match(/[A-Za-z0-9]/g) || []).length; | |
| const printable = (normalized.match(/[\x20-\x7E]/g) || []).length; | |
| const angleBrackets = (normalized.match(/[<>]/g) || []).length; | |
| const alphaNumRatio = alphaNum / normalized.length; | |
| const printableRatio = printable / normalized.length; | |
| const bracketRatio = angleBrackets / normalized.length; | |
| if (normalized.length >= 24 && printableRatio > 0.95 && alphaNumRatio < 0.18) { | |
| return true; | |
| } | |
| if (normalized.length >= 24 && bracketRatio > 0.25) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| function trimUnstableUtf8Tail(text) { | |
| if (typeof text !== "string" || text.length === 0) { | |
| return ""; | |
| } | |
| let end = text.length; | |
| while (end > 0 && text.charCodeAt(end - 1) === 65533) { | |
| end -= 1; | |
| } | |
| if (end > 0) { | |
| const tail = text.charCodeAt(end - 1); | |
| if (tail >= 55296 && tail <= 56319) { | |
| end -= 1; | |
| } | |
| } | |
| return end === text.length ? text : text.slice(0, end); | |
| } | |
| // js/src/runtime.ts | |
| var textEncoder = new TextEncoder(); | |
| var LlamaWebGpuBridgeRuntime = class { | |
| constructor(config = {}) { | |
| this._config = config; | |
| this._core = null; | |
| this._backendLabels = []; | |
| this._gpuActive = false; | |
| this._modelPath = null; | |
| this._modelPaths = []; | |
| this._modelBytes = 0; | |
| this._mmProjPath = null; | |
| this._mmSupportsVision = false; | |
| this._mmSupportsAudio = false; | |
| this._mediaFileCounter = 0; | |
| this._stateFileCounter = 0; | |
| this._decisionFileCounter = 0; | |
| this._loraFileCounter = 0; | |
| this._loraAdaptersLoaded = false; | |
| this._stagedMediaPaths = []; | |
| this._nCtx = 4096; | |
| this._abortRequested = false; | |
| this._textToSpeechActive = false; | |
| this._textToSpeechDone = null; | |
| this._resolveTextToSpeechDone = null; | |
| this._runtimeNotes = []; | |
| this._threadPoolSizeHint = Number(config.threadPoolSize) > 0 ? Math.max(1, Math.trunc(Number(config.threadPoolSize))) : null; | |
| const requestedThreads = Number(config.threads) > 0 ? Number(config.threads) : this._resolveAutoThreadCount(); | |
| this._threads = this._capThreadsToPool(requestedThreads); | |
| const requestedThreadsBatch = Number(config.threadsBatch) > 0 ? Number(config.threadsBatch) : this._threads; | |
| this._threadsBatch = this._capThreadsToPool( | |
| requestedThreadsBatch, | |
| { noteTag: "threads_batch_capped_pool" } | |
| ); | |
| this._nBatch = Number(config.nBatch) > 0 ? Math.max(32, Math.trunc(Number(config.nBatch))) : 0; | |
| this._nUbatch = Number(config.nUbatch) > 0 ? Math.max(32, Math.trunc(Number(config.nUbatch))) : 0; | |
| this._nGpuLayers = Number.isFinite(config.nGpuLayers) ? Number(config.nGpuLayers) : -1; | |
| this._nSeqMax = 0; | |
| this._useMmap = false; | |
| this._useMlock = false; | |
| this._flashAttention = -1; | |
| this._cacheTypeK = 1; | |
| this._cacheTypeV = 1; | |
| this._kvUnified = -1; | |
| this._ropeFrequencyBase = 0; | |
| this._ropeFrequencyScale = 0; | |
| this._splitMode = -1; | |
| this._mainGpu = -1; | |
| this._loadMtp = false; | |
| this._speculativeRollbackTokenMax = 0; | |
| this._isSafari = isSafariUserAgent(this._config.userAgent ?? globalThis.navigator?.userAgent ?? ""); | |
| this._coreVariant = "uninitialized"; | |
| this._preferMemory64 = this._config.preferMemory64 !== false; | |
| this._modelSource = "network"; | |
| this._modelCacheState = "disabled"; | |
| this._modelCacheName = defaultModelCacheName; | |
| this._loadedModelUrl = null; | |
| this._mmProjSourceUrl = null; | |
| this._draftModel = null; | |
| this._speculativeFileCounter = 0; | |
| this._suppressedWarmupWarningCount = 0; | |
| this._didReportWarmupWarningSuppression = false; | |
| this._remoteFetchThresholdBytes = Number(config.remoteFetchThresholdBytes) > 0 ? Number(config.remoteFetchThresholdBytes) : 1900 * 1024 * 1024; | |
| this._remoteFetchChunkBytes = Number(config.remoteFetchChunkBytes) > 0 ? Number(config.remoteFetchChunkBytes) : 16 * 1024 * 1024; | |
| this._mediaMaxImagePixels = Number(config.mediaMaxImagePixels) > 0 ? Math.max(65536, Math.min(33554432, Math.trunc(Number(config.mediaMaxImagePixels)))) : 1024 * 1024; | |
| this._mediaMaxImageEdge = Number(config.mediaMaxImageEdge) > 0 ? Math.max(64, Math.min(16384, Math.trunc(Number(config.mediaMaxImageEdge)))) : 1280; | |
| this._disableImageDownscale = config.disableImageDownscale === true; | |
| this._activeTransferAbortController = null; | |
| this._lastCoreErrorText = ""; | |
| this._lastCoreErrorHint = ""; | |
| this._logLevel = Number.isFinite(config.logLevel) ? Math.max(0, Math.min(4, Math.trunc(config.logLevel))) : 2; | |
| } | |
| static supportsSafariAdaptiveGpu = true; | |
| _pushRuntimeNote(note) { | |
| if (typeof note !== "string" || note.length === 0) { | |
| return; | |
| } | |
| if (!Array.isArray(this._runtimeNotes)) { | |
| this._runtimeNotes = []; | |
| } | |
| if (!this._runtimeNotes.includes(note)) { | |
| this._runtimeNotes.push(note); | |
| } | |
| } | |
| _detectThreadPoolSizeFromCore() { | |
| if (!this._coreSupportsPthreads()) { | |
| return 1; | |
| } | |
| const core = this._core; | |
| if (!core || typeof core !== "object") { | |
| return null; | |
| } | |
| try { | |
| if (typeof core.ccall === "function") { | |
| const compiledPoolSize = Number( | |
| core.ccall("llamadart_webgpu_pthread_pool_size", "number", [], []) | |
| ); | |
| if (Number.isFinite(compiledPoolSize) && compiledPoolSize > 0) { | |
| return Math.max(1, Math.trunc(compiledPoolSize)); | |
| } | |
| } | |
| } catch (_) { | |
| } | |
| try { | |
| const pThread = core.PThread; | |
| if (!pThread || typeof pThread !== "object") { | |
| return null; | |
| } | |
| const unused = Array.isArray(pThread.unusedWorkers) ? pThread.unusedWorkers.length : 0; | |
| const running = Array.isArray(pThread.runningWorkers) ? pThread.runningWorkers.length : 0; | |
| const total = unused + running; | |
| return total > 0 ? total : null; | |
| } catch (_) { | |
| return null; | |
| } | |
| } | |
| _coreSupportsPthreads() { | |
| const core = this._core; | |
| if (!core || typeof core !== "object") { | |
| return false; | |
| } | |
| try { | |
| if (typeof core.ccall === "function") { | |
| const compiledWithPthreads = Number( | |
| core.ccall("llamadart_webgpu_supports_pthreads", "number", [], []) | |
| ); | |
| if (Number.isFinite(compiledWithPthreads)) { | |
| return compiledWithPthreads !== 0; | |
| } | |
| } | |
| } catch (_) { | |
| } | |
| try { | |
| const wasmBuffer = core.wasmMemory?.buffer; | |
| if (typeof SharedArrayBuffer === "function" && wasmBuffer instanceof SharedArrayBuffer) { | |
| return true; | |
| } | |
| } catch (_) { | |
| } | |
| try { | |
| const heapBuffer = core.HEAP8?.buffer || core.HEAPU8?.buffer; | |
| if (typeof SharedArrayBuffer === "function" && heapBuffer instanceof SharedArrayBuffer) { | |
| return true; | |
| } | |
| } catch (_) { | |
| } | |
| try { | |
| const pThread = core.PThread; | |
| if (!pThread || typeof pThread !== "object") { | |
| return false; | |
| } | |
| return Array.isArray(pThread.unusedWorkers) || Array.isArray(pThread.runningWorkers) || typeof pThread.allocateUnusedWorker === "function"; | |
| } catch (_) { | |
| return false; | |
| } | |
| } | |
| _syncThreadPoolSizeHintFromCore() { | |
| const detected = this._detectThreadPoolSizeFromCore(); | |
| if (!Number.isFinite(detected) || detected <= 0) { | |
| return; | |
| } | |
| this._threadPoolSizeHint = Math.max(1, Math.trunc(detected)); | |
| } | |
| _resolveAutoThreadCount() { | |
| const hardwareThreads = Number(globalThis.navigator?.hardwareConcurrency); | |
| if (Number.isFinite(hardwareThreads) && hardwareThreads > 0) { | |
| return Math.max(1, Math.min(8, Math.trunc(hardwareThreads))); | |
| } | |
| return 4; | |
| } | |
| _capThreadsToPool(candidate, { noteTag = "threads_capped_pool" } = {}) { | |
| let resolved = Number(candidate); | |
| if (!Number.isFinite(resolved) || resolved <= 0) { | |
| resolved = 1; | |
| } | |
| resolved = Math.max(1, Math.trunc(resolved)); | |
| const poolSize = Number(this._threadPoolSizeHint); | |
| if (Number.isFinite(poolSize) && poolSize > 0 && resolved > poolSize) { | |
| if (noteTag) { | |
| this._pushRuntimeNote(`${noteTag}:${poolSize}`); | |
| } | |
| return poolSize; | |
| } | |
| return resolved; | |
| } | |
| _isVerboseWarmupWarning(text) { | |
| const lowered = String(text || "").toLowerCase(); | |
| if (lowered.length === 0) { | |
| return false; | |
| } | |
| if (lowered.includes("warmup:")) { | |
| return true; | |
| } | |
| if (lowered.includes("please report this on github as an issue")) { | |
| return true; | |
| } | |
| if (lowered.includes("warning: ref:")) { | |
| return true; | |
| } | |
| if (lowered.includes("github.com/ggml-org/llama.cpp/pull/")) { | |
| return true; | |
| } | |
| if (lowered.includes("****************")) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| _emitSuppressedWarmupWarningSummaryIfNeeded() { | |
| if (this._suppressedWarmupWarningCount <= 0) { | |
| return; | |
| } | |
| if (this._logLevel <= 2) { | |
| this._emitLogger( | |
| "log", | |
| `info: suppressed ${this._suppressedWarmupWarningCount} verbose warmup log lines (set bridge/runtime log level to Debug to inspect full warmup trace).` | |
| ); | |
| } | |
| this._suppressedWarmupWarningCount = 0; | |
| } | |
| _shouldAttemptGenerationRecovery(errorText, options = {}, generated = 0) { | |
| if (generated > 0) { | |
| return false; | |
| } | |
| if (this._nGpuLayers <= 0) { | |
| return false; | |
| } | |
| if (this._coreVariant !== "wasm64") { | |
| return false; | |
| } | |
| if (!hasModelSource(this._loadedModelUrl)) { | |
| return false; | |
| } | |
| if (options._llamadartGenerationRecoveryAttempted === true) { | |
| return false; | |
| } | |
| if (Array.isArray(options.parts) && options.parts.length > 0) { | |
| return false; | |
| } | |
| const lowered = String(errorText || "").toLowerCase(); | |
| if (lowered.includes("failed to decode")) { | |
| return true; | |
| } | |
| if (lowered.includes("failed to compute graph")) { | |
| return true; | |
| } | |
| if (lowered.includes("ggml_backend_sched_graph_compute_async failed")) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| _isContextLimitGenerationError(errorText = "") { | |
| const lowered = [ | |
| String(errorText || ""), | |
| String(this._lastCoreErrorText || ""), | |
| String(this._lastCoreErrorHint || "") | |
| ].join(" ").toLowerCase(); | |
| return lowered.includes("failed to find a memory slot for batch") || lowered.includes("failed to prepare attention batches") || lowered.includes("context overflow") || lowered.includes("context full") || lowered.includes("kv cache full") || lowered.includes("insufficient kv") || lowered.includes("no kv slot"); | |
| } | |
| async _recoverGenerationWithCpuFallback(options = {}) { | |
| const modelUrl = cloneModelSource(this._loadedModelUrl); | |
| if (!hasModelSource(modelUrl)) { | |
| return false; | |
| } | |
| if (this._loraAdaptersLoaded) { | |
| this._runtimeNotes.push("generation_recovery_cpu_skipped_lora"); | |
| return false; | |
| } | |
| this._runtimeNotes.push("generation_recovery_cpu_attempt"); | |
| this._emitLogger( | |
| "warn", | |
| "warning: generation failed on wasm64/WebGPU; retrying by reloading model with CPU fallback for stability." | |
| ); | |
| const previousPreferMemory64 = this._preferMemory64; | |
| const previousMMProjSourceUrl = this._mmProjSourceUrl; | |
| const previousDraftModel = this._draftModel; | |
| const resolvedLoadState = this._resolvedLoadState(); | |
| try { | |
| this._preferMemory64 = false; | |
| await this.loadModelFromUrl(modelUrl, { | |
| nCtx: this._nCtx, | |
| nThreads: this._threads, | |
| nGpuLayers: 0, | |
| useCache: true, | |
| forceRemoteFetchBackend: false, | |
| remoteFetchChunkBytes: this._resolveRemoteFetchChunkBytes(options), | |
| safariGpuProbe: false, | |
| signal: options.signal | |
| }, resolvedLoadState); | |
| if (typeof previousMMProjSourceUrl === "string" && previousMMProjSourceUrl.length > 0) { | |
| try { | |
| await this.loadMultimodalProjector(previousMMProjSourceUrl); | |
| } catch (_) { | |
| this._runtimeNotes.push("generation_recovery_mmproj_reload_failed"); | |
| } | |
| } | |
| if (previousDraftModel) { | |
| try { | |
| await this.loadDraftModel(previousDraftModel.url, previousDraftModel.options); | |
| } catch (_) { | |
| this._runtimeNotes.push("generation_recovery_draft_reload_failed"); | |
| } | |
| } | |
| this._runtimeNotes.push("generation_recovery_cpu_applied"); | |
| return true; | |
| } catch (_) { | |
| this._runtimeNotes.push("generation_recovery_cpu_failed"); | |
| return false; | |
| } finally { | |
| this._preferMemory64 = previousPreferMemory64; | |
| } | |
| } | |
| _loggerFor(level) { | |
| const logger = this._config?.logger; | |
| const fallback = typeof console !== "undefined" ? console : null; | |
| if (logger && typeof logger[level] === "function") { | |
| return logger[level].bind(logger); | |
| } | |
| if (!fallback) { | |
| return () => { | |
| }; | |
| } | |
| if (typeof fallback[level] === "function") { | |
| return fallback[level].bind(fallback); | |
| } | |
| if (typeof fallback.log === "function") { | |
| return fallback.log.bind(fallback); | |
| } | |
| return () => { | |
| }; | |
| } | |
| _shouldEmitLoggerLevel(level) { | |
| const current = Number(this._logLevel); | |
| if (!Number.isFinite(current) || current < 0) { | |
| return true; | |
| } | |
| const threshold = logThresholdForConfiguredLevel( | |
| Math.max(0, Math.min(4, Math.trunc(current))) | |
| ); | |
| if (threshold > 3) { | |
| return false; | |
| } | |
| return logLevelForName(level) >= threshold; | |
| } | |
| _emitLogger(level, message) { | |
| if (!this._shouldEmitLoggerLevel(level)) { | |
| return; | |
| } | |
| try { | |
| this._loggerFor(level)(message); | |
| } catch (_) { | |
| } | |
| } | |
| _classifyCoreErrorLine(text) { | |
| const trimmed = String(text ?? "").trim(); | |
| if (trimmed.length === 0) { | |
| return "ignore"; | |
| } | |
| if (this._isVerboseWarmupWarning(trimmed)) { | |
| return "warmup"; | |
| } | |
| const lowered = trimmed.toLowerCase(); | |
| if (lowered.startsWith("warning") || lowered.startsWith("warn:") || lowered.includes(" warning:")) { | |
| return "warn"; | |
| } | |
| if (lowered.startsWith("error") || lowered.startsWith("err:") || lowered.includes(" error:") || lowered.includes("failed") || lowered.includes("exception") || lowered.includes("abort") || lowered.includes("fatal") || lowered.includes("out of memory") || lowered.includes("invalid")) { | |
| return "error"; | |
| } | |
| return "info"; | |
| } | |
| _applyCoreLogLevel() { | |
| if (!this._core) { | |
| return; | |
| } | |
| try { | |
| this._core.ccall( | |
| "llamadart_webgpu_set_log_level", | |
| null, | |
| ["number"], | |
| [this._logLevel] | |
| ); | |
| } catch (_) { | |
| } | |
| } | |
| _coreErrorMessage(prefix, fallbackCode = 0) { | |
| try { | |
| const err = this._core?.ccall("llamadart_webgpu_last_error", "string", [], []); | |
| if (err) { | |
| return `${prefix}: ${err}`; | |
| } | |
| } catch (_) { | |
| } | |
| return `${prefix} (code=${fallbackCode})`; | |
| } | |
| _resolveCacheName(options = {}) { | |
| if (typeof options.cacheName === "string" && options.cacheName.trim().length > 0) { | |
| return options.cacheName.trim(); | |
| } | |
| if (typeof this._config.cacheName === "string" && this._config.cacheName.trim().length > 0) { | |
| return this._config.cacheName.trim(); | |
| } | |
| return defaultModelCacheName; | |
| } | |
| _isWorkerRuntime() { | |
| return typeof WorkerGlobalScope !== "undefined" && globalThis instanceof WorkerGlobalScope; | |
| } | |
| _resolveRemoteFetchThresholdBytes(options = {}) { | |
| const candidate = Number(options.remoteFetchThresholdBytes); | |
| if (Number.isFinite(candidate) && candidate > 0) { | |
| return Math.trunc(candidate); | |
| } | |
| return Math.trunc(this._remoteFetchThresholdBytes); | |
| } | |
| _resolveRemoteFetchChunkBytes(options = {}) { | |
| const candidate = Number(options.remoteFetchChunkBytes); | |
| if (Number.isFinite(candidate) && candidate > 0) { | |
| return Math.max(16 * 1024, Math.trunc(candidate)); | |
| } | |
| return Math.max(16 * 1024, Math.trunc(this._remoteFetchChunkBytes)); | |
| } | |
| _canUseRemoteFetchBackend(options = {}) { | |
| if (options.forceRemoteFetchBackend === false) { | |
| return false; | |
| } | |
| if (!this._isWorkerRuntime()) { | |
| return false; | |
| } | |
| if (options.forceRemoteFetchBackend === true) { | |
| return true; | |
| } | |
| return this._config.allowAutoRemoteFetchBackend === true; | |
| } | |
| async _tryHeadContentLength(url) { | |
| function parseSizeFromHeaders(headers) { | |
| if (!headers) { | |
| return 0; | |
| } | |
| const length = Number(headers.get("content-length")) || 0; | |
| if (length > 0) { | |
| return length; | |
| } | |
| const linkedSize = Number(headers.get("x-linked-size")) || 0; | |
| if (linkedSize > 0) { | |
| return linkedSize; | |
| } | |
| const contentRange = String(headers.get("content-range") || ""); | |
| const slash = contentRange.lastIndexOf("/"); | |
| if (slash >= 0 && slash + 1 < contentRange.length) { | |
| const total = Number(contentRange.slice(slash + 1)) || 0; | |
| if (total > 0) { | |
| return total; | |
| } | |
| } | |
| return 0; | |
| } | |
| try { | |
| const response = await this._fetchWithTimeout(url, { | |
| method: "HEAD", | |
| cache: "no-store" | |
| }, 45e3); | |
| if (!response.ok) { | |
| throw new Error("HEAD request failed"); | |
| } | |
| const size = parseSizeFromHeaders(response.headers); | |
| if (size > 0) { | |
| return size; | |
| } | |
| } catch (_) { | |
| } | |
| try { | |
| const probe = await this._fetchWithTimeout(url, { | |
| headers: { | |
| Range: "bytes=0-0" | |
| }, | |
| cache: "no-store" | |
| }, 45e3); | |
| if (!(probe.ok || probe.status === 206)) { | |
| return null; | |
| } | |
| const size = parseSizeFromHeaders(probe.headers); | |
| if (size > 0) { | |
| this._runtimeNotes.push("model_fetch_size_probe"); | |
| return size; | |
| } | |
| } catch (_) { | |
| } | |
| return null; | |
| } | |
| async _resolveRemoteFetchUrl(url) { | |
| function parseSizeFromHeaders(headers) { | |
| if (!headers) { | |
| return 0; | |
| } | |
| const contentRange = String(headers.get("content-range") || ""); | |
| const slash = contentRange.lastIndexOf("/"); | |
| if (slash >= 0 && slash + 1 < contentRange.length) { | |
| const total = Number(contentRange.slice(slash + 1)) || 0; | |
| if (total > 0) { | |
| return total; | |
| } | |
| } | |
| const linkedSize = Number(headers.get("x-linked-size")) || 0; | |
| if (linkedSize > 0) { | |
| return linkedSize; | |
| } | |
| const length = Number(headers.get("content-length")) || 0; | |
| if (length > 0) { | |
| return length; | |
| } | |
| return 0; | |
| } | |
| try { | |
| const probe = await this._fetchWithTimeout(url, { | |
| headers: { | |
| Range: "bytes=0-0" | |
| }, | |
| cache: "no-store" | |
| }, 45e3); | |
| if (!(probe.ok || probe.status === 206)) { | |
| return null; | |
| } | |
| const resolvedUrl = typeof probe.url === "string" && probe.url.length > 0 ? probe.url : null; | |
| const sizeBytes = parseSizeFromHeaders(probe.headers); | |
| return { | |
| resolvedUrl, | |
| sizeBytes: sizeBytes > 0 ? sizeBytes : null | |
| }; | |
| } catch (_) { | |
| return null; | |
| } | |
| } | |
| _resolveNativeLoadOptions(options = {}) { | |
| this._nSeqMax = parsePositiveInteger(options.nSeqMax); | |
| this._useMmap = parseBooleanFlag(options.useMmap, false); | |
| this._useMlock = parseBooleanFlag(options.useMlock, false); | |
| this._flashAttention = parseEnumValue(options.flashAttention, [-1, 0, 1], -1); | |
| this._cacheTypeK = parseEnumValue(options.cacheTypeK, [1, 2, 8], 1); | |
| this._cacheTypeV = parseEnumValue(options.cacheTypeV, [1, 2, 8], 1); | |
| this._kvUnified = parseOptionalBooleanFlag(options.kvUnified); | |
| this._ropeFrequencyBase = parsePositiveNumber(options.ropeFrequencyBase); | |
| this._ropeFrequencyScale = parsePositiveNumber(options.ropeFrequencyScale); | |
| this._splitMode = parseEnumValue(options.splitMode, [0, 1, 2, 3], -1); | |
| this._mainGpu = parseInteger(options.mainGpu, -1); | |
| if (this._mainGpu < 0) { | |
| this._mainGpu = -1; | |
| } | |
| this._loadMtp = parseBooleanFlag(options.loadMtp, false); | |
| this._speculativeRollbackTokenMax = parsePositiveInteger(options.speculativeRollbackTokenMax); | |
| const wantsQuantizedKvCache = this._cacheTypeK !== 1 || this._cacheTypeV !== 1; | |
| if (this._flashAttention === 0 && wantsQuantizedKvCache) { | |
| throw new Error( | |
| "Non-F16 KV cache requires flashAttention to be auto or enabled." | |
| ); | |
| } | |
| if (this._flashAttention === -1 && wantsQuantizedKvCache) { | |
| this._flashAttention = 1; | |
| this._runtimeNotes.push("flash_attention:auto_enabled_for_kv_cache"); | |
| } | |
| if (this._kvUnified < 0 && this._nSeqMax > 1) { | |
| this._kvUnified = 1; | |
| this._runtimeNotes.push("kv_unified:auto_enabled_for_sequences"); | |
| } | |
| } | |
| _nativeLoadOptionValues() { | |
| return [ | |
| this._nSeqMax, | |
| this._useMmap ? 1 : 0, | |
| this._useMlock ? 1 : 0, | |
| this._flashAttention, | |
| this._cacheTypeK, | |
| this._cacheTypeV, | |
| this._kvUnified, | |
| this._ropeFrequencyBase, | |
| this._ropeFrequencyScale, | |
| this._splitMode, | |
| this._mainGpu, | |
| this._loadMtp ? 1 : 0, | |
| this._speculativeRollbackTokenMax | |
| ]; | |
| } | |
| _nativeLoadOptionTypes() { | |
| return this._nativeLoadOptionValues().map(() => "number"); | |
| } | |
| _resolvedLoadState() { | |
| return { | |
| _nCtx: this._nCtx, | |
| _threads: this._threads, | |
| _threadsBatch: this._threadsBatch, | |
| _nBatch: this._nBatch, | |
| _nUbatch: this._nUbatch, | |
| _nSeqMax: this._nSeqMax, | |
| _useMmap: this._useMmap, | |
| _useMlock: this._useMlock, | |
| _flashAttention: this._flashAttention, | |
| _cacheTypeK: this._cacheTypeK, | |
| _cacheTypeV: this._cacheTypeV, | |
| _kvUnified: this._kvUnified, | |
| _ropeFrequencyBase: this._ropeFrequencyBase, | |
| _ropeFrequencyScale: this._ropeFrequencyScale, | |
| _splitMode: this._splitMode, | |
| _mainGpu: this._mainGpu, | |
| _loadMtp: this._loadMtp, | |
| _speculativeRollbackTokenMax: this._speculativeRollbackTokenMax | |
| }; | |
| } | |
| async _tryLoadModelFromRemoteFetchBackend(core, url, options = {}) { | |
| if (!this._canUseRemoteFetchBackend(options)) { | |
| return { loaded: false, sizeBytes: null }; | |
| } | |
| const thresholdBytes = this._resolveRemoteFetchThresholdBytes(options); | |
| const chunkBytes = this._resolveRemoteFetchChunkBytes(options); | |
| const forceRemote = options.forceRemoteFetchBackend === true; | |
| let sizeBytes = Number(options.modelBytesHint); | |
| if (!Number.isFinite(sizeBytes) || sizeBytes <= 0) { | |
| sizeBytes = await this._tryHeadContentLength(url); | |
| } | |
| if (!forceRemote) { | |
| if (Number.isFinite(sizeBytes) && sizeBytes > 0 && sizeBytes < thresholdBytes) { | |
| this._runtimeNotes.push("model_fetch_backend_skipped_small"); | |
| return { loaded: false, sizeBytes }; | |
| } | |
| if (!Number.isFinite(sizeBytes) || sizeBytes <= 0) { | |
| this._runtimeNotes.push("model_fetch_backend_size_unknown"); | |
| this._runtimeNotes.push("model_fetch_backend_unknown_size_attempt"); | |
| } | |
| } | |
| this._runtimeNotes.push("model_fetch_backend_attempt"); | |
| this._runtimeNotes.push(`model_fetch_chunk:${chunkBytes}`); | |
| this._lastCoreErrorHint = ""; | |
| let remoteFetchUrl = url; | |
| const resolvedProbe = await this._resolveRemoteFetchUrl(url); | |
| if (resolvedProbe?.resolvedUrl) { | |
| remoteFetchUrl = resolvedProbe.resolvedUrl; | |
| if (remoteFetchUrl !== url) { | |
| this._runtimeNotes.push("model_fetch_backend_resolved_url"); | |
| } | |
| } | |
| if ((!Number.isFinite(sizeBytes) || sizeBytes <= 0) && Number.isFinite(resolvedProbe?.sizeBytes) && resolvedProbe.sizeBytes > 0) { | |
| sizeBytes = resolvedProbe.sizeBytes; | |
| } | |
| try { | |
| globalThis.__llamadartFetchBackendLastError = null; | |
| } catch (_) { | |
| } | |
| if (typeof options.progressCallback === "function") { | |
| options.progressCallback({ loaded: 0, total: Number.isFinite(sizeBytes) ? sizeBytes : 0 }); | |
| } | |
| try { | |
| const rc = Number( | |
| await core.ccall( | |
| "llamadart_webgpu_load_model_from_url", | |
| "number", | |
| [ | |
| "string", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| ...this._nativeLoadOptionTypes() | |
| ], | |
| [ | |
| remoteFetchUrl, | |
| this._nCtx, | |
| this._threads, | |
| this._threadsBatch, | |
| this._nBatch, | |
| this._nUbatch, | |
| this._nGpuLayers, | |
| chunkBytes, | |
| ...this._nativeLoadOptionValues() | |
| ], | |
| { async: true } | |
| ) | |
| ); | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Fetch-backed model load failed", rc)); | |
| } | |
| this._modelSource = "network-fetch"; | |
| this._modelPath = null; | |
| this._modelBytes = Number.isFinite(sizeBytes) && sizeBytes > 0 ? Math.trunc(sizeBytes) : 1; | |
| this._runtimeNotes.push("model_source_fetch_backend"); | |
| if (typeof options.progressCallback === "function") { | |
| const resolved = Number.isFinite(sizeBytes) && sizeBytes > 0 ? sizeBytes : 1; | |
| options.progressCallback({ loaded: resolved, total: resolved }); | |
| } | |
| return { | |
| loaded: true, | |
| sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null | |
| }; | |
| } catch (error) { | |
| const text = String(error || "").toLowerCase(); | |
| if (text.includes("aborted(native code called abort())")) { | |
| try { | |
| const stderrText = String( | |
| this._lastCoreErrorHint || this._lastCoreErrorText || "" | |
| ).trim(); | |
| if (stderrText.length > 0) { | |
| const token = stderrText.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, ""); | |
| if (token.length > 0) { | |
| this._runtimeNotes.push(`model_core_stderr:${token}`); | |
| } | |
| } | |
| } catch (_) { | |
| } | |
| try { | |
| const fetchError = String( | |
| globalThis.__llamadartFetchBackendLastError || "" | |
| ).trim(); | |
| if (fetchError.length > 0) { | |
| const token = fetchError.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, ""); | |
| if (token.length > 0) { | |
| this._runtimeNotes.push(`model_fetch_js_error:${token}`); | |
| } | |
| } | |
| } catch (_) { | |
| } | |
| try { | |
| const stats = globalThis.__llamadartFetchBackendStats; | |
| if (stats && typeof stats === "object") { | |
| const reads = Number(stats.reads) || 0; | |
| const getSize = Number(stats.getSize) || 0; | |
| const ranges = Number(stats.ranges) || 0; | |
| const fallbacks = Number(stats.wholeFileFallbacks) || 0; | |
| const errors = Number(stats.errors) || 0; | |
| this._runtimeNotes.push( | |
| `model_fetch_stats:r${reads}_s${getSize}_q${ranges}_f${fallbacks}_e${errors}` | |
| ); | |
| } | |
| } catch (_) { | |
| } | |
| try { | |
| const coreError = String( | |
| this._core?.ccall("llamadart_webgpu_last_error", "string", [], []) || "" | |
| ).trim(); | |
| if (coreError.length > 0) { | |
| const token = coreError.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, ""); | |
| if (token.length > 0) { | |
| this._runtimeNotes.push(`model_fetch_core_error:${token}`); | |
| } | |
| } | |
| } catch (_) { | |
| } | |
| this._runtimeNotes.push("model_fetch_backend_abort"); | |
| throw error; | |
| } | |
| if (text.includes("fetch-backed model load failed")) { | |
| this._runtimeNotes.push("model_fetch_backend_failed"); | |
| return { | |
| loaded: false, | |
| sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null | |
| }; | |
| } | |
| if (text.includes("load_model_from_url") && (text.includes("not found") || text.includes("undefined symbol") || text.includes("missing function"))) { | |
| this._runtimeNotes.push("model_fetch_backend_unavailable"); | |
| return { | |
| loaded: false, | |
| sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null | |
| }; | |
| } | |
| if (text.includes("worker-thread bridge runtime")) { | |
| this._runtimeNotes.push("model_fetch_backend_requires_worker"); | |
| return { | |
| loaded: false, | |
| sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null | |
| }; | |
| } | |
| throw error; | |
| } | |
| } | |
| _beginTransferAbortController() { | |
| if (typeof AbortController !== "function") { | |
| this._activeTransferAbortController = null; | |
| return null; | |
| } | |
| if (this._activeTransferAbortController) { | |
| try { | |
| this._activeTransferAbortController.abort(); | |
| } catch (_) { | |
| } | |
| } | |
| const controller = new AbortController(); | |
| this._activeTransferAbortController = controller; | |
| return controller; | |
| } | |
| _clearTransferAbortController(controller) { | |
| if (this._activeTransferAbortController === controller) { | |
| this._activeTransferAbortController = null; | |
| } | |
| } | |
| _resolveFetchTimeoutMs(options = {}, defaultTimeoutMs = 18e4) { | |
| const configured = Number(options.fetchTimeoutMs); | |
| if (Number.isFinite(configured) && configured > 0) { | |
| return Math.max(1e4, Math.min(18e5, Math.trunc(configured))); | |
| } | |
| return defaultTimeoutMs; | |
| } | |
| _resolveStreamChunkTimeoutMs(options = {}, defaultTimeoutMs = 9e4) { | |
| const configured = Number(options.streamChunkTimeoutMs); | |
| if (Number.isFinite(configured) && configured > 0) { | |
| return Math.max(5e3, Math.min(3e5, Math.trunc(configured))); | |
| } | |
| return defaultTimeoutMs; | |
| } | |
| _resolveCoreInitTimeoutMs() { | |
| const configured = Number(this._config.coreInitTimeoutMs); | |
| if (!Number.isFinite(configured) || configured <= 0) { | |
| return 9e4; | |
| } | |
| return Math.max(1e4, Math.min(6e5, Math.trunc(configured))); | |
| } | |
| _resolveMediaImageMaxPixels(options = {}) { | |
| if (options.disableImageDownscale === true || this._disableImageDownscale) { | |
| return 0; | |
| } | |
| const configured = Number(options.mediaMaxImagePixels); | |
| if (Number.isFinite(configured) && configured > 0) { | |
| return Math.max(65536, Math.min(33554432, Math.trunc(configured))); | |
| } | |
| return this._mediaMaxImagePixels; | |
| } | |
| _resolveMediaImageMaxEdge(options = {}) { | |
| if (options.disableImageDownscale === true || this._disableImageDownscale) { | |
| return 0; | |
| } | |
| const configured = Number(options.mediaMaxImageEdge); | |
| if (Number.isFinite(configured) && configured > 0) { | |
| return Math.max(64, Math.min(16384, Math.trunc(configured))); | |
| } | |
| return this._mediaMaxImageEdge; | |
| } | |
| async _fetchWithTimeout(url, init = {}, timeoutMs = 0) { | |
| const resolvedTimeout = Number(timeoutMs); | |
| if (!Number.isFinite(resolvedTimeout) || resolvedTimeout <= 0) { | |
| return fetch(url, init); | |
| } | |
| if (typeof AbortController !== "function") { | |
| return Promise.race([ | |
| fetch(url, init), | |
| new Promise((_, reject) => { | |
| globalThis.setTimeout( | |
| () => reject(new Error(`fetch timeout (${resolvedTimeout}ms)`)), | |
| resolvedTimeout | |
| ); | |
| }) | |
| ]); | |
| } | |
| const timeoutController = new AbortController(); | |
| const externalSignal = init.signal; | |
| let didTimeout = false; | |
| let timeoutHandle = null; | |
| const onExternalAbort = () => { | |
| try { | |
| timeoutController.abort(); | |
| } catch (_) { | |
| } | |
| }; | |
| if (externalSignal && typeof externalSignal.addEventListener === "function") { | |
| externalSignal.addEventListener("abort", onExternalAbort, { once: true }); | |
| if (externalSignal.aborted) { | |
| onExternalAbort(); | |
| } | |
| } | |
| timeoutHandle = globalThis.setTimeout(() => { | |
| didTimeout = true; | |
| try { | |
| timeoutController.abort(); | |
| } catch (_) { | |
| } | |
| }, resolvedTimeout); | |
| try { | |
| return await fetch(url, { | |
| ...init, | |
| signal: timeoutController.signal | |
| }); | |
| } catch (error) { | |
| if (didTimeout) { | |
| throw new Error(`fetch timeout (${resolvedTimeout}ms)`); | |
| } | |
| throw error; | |
| } finally { | |
| if (timeoutHandle != null) { | |
| globalThis.clearTimeout(timeoutHandle); | |
| } | |
| if (externalSignal && typeof externalSignal.removeEventListener === "function") { | |
| externalSignal.removeEventListener("abort", onExternalAbort); | |
| } | |
| } | |
| } | |
| _unlinkDecisionFile(path) { | |
| if (!this._core || typeof path !== "string" || path.length === 0) { | |
| return; | |
| } | |
| try { | |
| this._core.FS.unlink(path); | |
| } catch (_) { | |
| } | |
| } | |
| _deleteFsFile(path) { | |
| if (!this._core || typeof path !== "string" || path.length === 0) { | |
| return false; | |
| } | |
| return unlinkFsFile(this._core.FS, path); | |
| } | |
| _releaseModelFiles() { | |
| const paths = Array.isArray(this._modelPaths) && this._modelPaths.length > 0 ? [...this._modelPaths] : this._modelPath ? [this._modelPath] : []; | |
| let removed = 0; | |
| for (const path of paths) { | |
| if (this._deleteFsFile(path)) { | |
| removed += 1; | |
| } | |
| } | |
| this._modelPaths = []; | |
| return removed; | |
| } | |
| // A load replaces the current model. Free it, its projector and their FS files | |
| // before downloading the next model so both never occupy the wasm heap at once. | |
| // The core refuses while a generation or speech synthesis is still active, | |
| // which leaves the current model untouched. | |
| _releaseLoadedModel(core) { | |
| const projectorPath = this._mmProjPath; | |
| const hasModelFiles = Array.isArray(this._modelPaths) && this._modelPaths.length > 0; | |
| if (this._modelBytes <= 0 && !projectorPath && !hasModelFiles) { | |
| return false; | |
| } | |
| const rc = Number(core.ccall("llamadart_webgpu_free_model", "number", [], [])); | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to release the loaded model", rc)); | |
| } | |
| this._loraAdaptersLoaded = false; | |
| this._clearStagedMediaFiles(); | |
| this._deleteFsFile(projectorPath); | |
| this._releaseModelFiles(); | |
| this._modelPath = null; | |
| this._modelBytes = 0; | |
| this._modelSource = "network"; | |
| this._mmProjPath = null; | |
| this._mmSupportsVision = false; | |
| this._mmSupportsAudio = false; | |
| this._draftModel = null; | |
| this._runtimeNotes.push("previous_model_released"); | |
| return true; | |
| } | |
| async _getCachedModelResponse(url, options = {}) { | |
| let useCache = options.useCache !== false; | |
| const forceRefresh = options.force === true; | |
| const requireReadableStream = options.requireReadableStream === true; | |
| const requestHeaders = options.requestHeaders && typeof options.requestHeaders === "object" ? options.requestHeaders : null; | |
| if (requestHeaders && Object.keys(requestHeaders).length > 0) { | |
| useCache = false; | |
| } | |
| const fetchOptions = { | |
| ...options.signal ? { signal: options.signal } : {}, | |
| ...requestHeaders ? { headers: requestHeaders } : {} | |
| }; | |
| const fetchTimeoutMs = this._resolveFetchTimeoutMs(options, 18e4); | |
| this._modelSource = "network"; | |
| this._modelCacheState = useCache ? "unavailable" : "disabled"; | |
| this._modelCacheName = this._resolveCacheName(options); | |
| if (!useCache) { | |
| const response = await this._fetchWithTimeout(url, { | |
| cache: "no-store", | |
| ...fetchOptions | |
| }, fetchTimeoutMs); | |
| this._modelCacheState = "disabled"; | |
| if (requireReadableStream) { | |
| this._runtimeNotes.push( | |
| hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream" | |
| ); | |
| } | |
| return response; | |
| } | |
| if (!globalThis.caches || typeof globalThis.caches.open !== "function") { | |
| this._modelCacheState = "unavailable"; | |
| const response = await this._fetchWithTimeout(url, fetchOptions, fetchTimeoutMs); | |
| if (requireReadableStream) { | |
| this._runtimeNotes.push( | |
| hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream" | |
| ); | |
| } | |
| return response; | |
| } | |
| const cacheKey = normalizeAbsoluteUrl(url); | |
| try { | |
| const cache = await globalThis.caches.open(this._modelCacheName); | |
| const cached = forceRefresh ? null : await cache.match(cacheKey); | |
| if (cached) { | |
| const cacheHasStream = hasReadableResponseStream(cached); | |
| if (!requireReadableStream || cacheHasStream) { | |
| this._modelSource = "cache"; | |
| this._modelCacheState = "hit"; | |
| this._runtimeNotes.push("model_cache_hit"); | |
| if (requireReadableStream) { | |
| this._runtimeNotes.push("model_cache_stream"); | |
| } | |
| return cached; | |
| } | |
| this._runtimeNotes.push("model_cache_hit_no_stream"); | |
| this._modelSource = "network"; | |
| this._modelCacheState = "refresh"; | |
| const refreshed = await this._fetchWithTimeout(url, { | |
| cache: "no-store", | |
| ...fetchOptions | |
| }, fetchTimeoutMs); | |
| if (refreshed.ok) { | |
| try { | |
| await cache.put(cacheKey, refreshed.clone()); | |
| this._modelCacheState = "stored"; | |
| this._runtimeNotes.push("model_cache_stored"); | |
| } catch (_) { | |
| this._modelCacheState = "store_failed"; | |
| this._runtimeNotes.push("model_cache_store_failed"); | |
| } | |
| } | |
| this._runtimeNotes.push( | |
| hasReadableResponseStream(refreshed) ? "model_network_stream" : "model_network_no_stream" | |
| ); | |
| return refreshed; | |
| } | |
| this._modelCacheState = forceRefresh ? "refresh" : "miss"; | |
| const response = await this._fetchWithTimeout(url, fetchOptions, fetchTimeoutMs); | |
| if (response.ok) { | |
| try { | |
| await cache.put(cacheKey, response.clone()); | |
| this._modelCacheState = "stored"; | |
| this._runtimeNotes.push("model_cache_stored"); | |
| } catch (_) { | |
| this._modelCacheState = "store_failed"; | |
| this._runtimeNotes.push("model_cache_store_failed"); | |
| } | |
| } | |
| if (requireReadableStream) { | |
| this._runtimeNotes.push( | |
| hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream" | |
| ); | |
| } | |
| return response; | |
| } catch (_) { | |
| this._modelCacheState = "error"; | |
| this._runtimeNotes.push("model_cache_error"); | |
| const response = await this._fetchWithTimeout(url, fetchOptions, fetchTimeoutMs); | |
| if (requireReadableStream) { | |
| this._runtimeNotes.push( | |
| hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream" | |
| ); | |
| } | |
| return response; | |
| } | |
| } | |
| async prefetchModelToCache(url, options = {}) { | |
| const useCache = options.useCache !== false; | |
| this._modelSource = "network"; | |
| this._modelCacheState = useCache ? "unavailable" : "disabled"; | |
| this._modelCacheName = this._resolveCacheName(options); | |
| const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null; | |
| const modelUrls = expandModelShardUrls(url); | |
| if (modelUrls.length === 0) { | |
| throw new Error("Model URL is empty."); | |
| } | |
| if (modelUrls.length > 1) { | |
| this._runtimeNotes.push(`model_split_cache_prefetch:${modelUrls.length}`); | |
| } | |
| const shardLoaded = new Array(modelUrls.length).fill(0); | |
| const shardTotals = new Array(modelUrls.length).fill(0); | |
| const emitAggregateProgress = () => { | |
| if (!progressCallback) { | |
| return; | |
| } | |
| const loaded = sumProgressValues(shardLoaded); | |
| const total = sumProgressValues(shardTotals); | |
| progressCallback({ | |
| loaded, | |
| total: total > 0 ? total : loaded | |
| }); | |
| }; | |
| const controller = this._beginTransferAbortController(); | |
| try { | |
| const fetchOptions = { | |
| cache: "no-store", | |
| ...controller?.signal ? { signal: controller.signal } : {} | |
| }; | |
| const fetchTimeoutMs = this._resolveFetchTimeoutMs(options, 18e4); | |
| const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs(options, 9e4); | |
| const cache = useCache && globalThis.caches && typeof globalThis.caches.open === "function" ? await globalThis.caches.open(this._modelCacheName) : null; | |
| if (!useCache || !cache) { | |
| this._modelCacheState = useCache ? "unavailable" : "disabled"; | |
| } | |
| for (let shardIndex = 0; shardIndex < modelUrls.length; shardIndex += 1) { | |
| const shardUrl = modelUrls[shardIndex]; | |
| const cacheKey = normalizeAbsoluteUrl(shardUrl); | |
| let response; | |
| let headerTotal = 0; | |
| if (cache) { | |
| const cached = options.force === true ? null : await cache.match(cacheKey); | |
| if (cached) { | |
| this._modelSource = "cache"; | |
| this._modelCacheState = "hit"; | |
| this._runtimeNotes.push("model_cache_hit"); | |
| headerTotal = Number(cached.headers.get("content-length")) || 0; | |
| const resolved = headerTotal > 0 ? headerTotal : 1; | |
| shardLoaded[shardIndex] = resolved; | |
| shardTotals[shardIndex] = resolved; | |
| emitAggregateProgress(); | |
| continue; | |
| } | |
| this._modelSource = "network"; | |
| this._modelCacheState = options.force === true ? "refresh" : "miss"; | |
| } | |
| response = await this._fetchWithTimeout(shardUrl, fetchOptions, fetchTimeoutMs); | |
| if (!response.ok) { | |
| throw new Error( | |
| `Failed to prefetch model shard: ${response.status} ${response.statusText}` | |
| ); | |
| } | |
| headerTotal = Number(response.headers.get("content-length")) || 0; | |
| if (headerTotal > 0) { | |
| shardTotals[shardIndex] = headerTotal; | |
| } | |
| const putPromise = cache ? cache.put(cacheKey, response.clone()).then(() => ({ ok: true, error: null })).catch((error) => ({ ok: false, error })) : null; | |
| await drainResponseWithProgress( | |
| response, | |
| progressCallback ? (progress) => { | |
| const loaded = Number(progress?.loaded) || 0; | |
| const total = Number(progress?.total) || 0; | |
| shardLoaded[shardIndex] = loaded; | |
| if (total > 0) { | |
| shardTotals[shardIndex] = total; | |
| } | |
| emitAggregateProgress(); | |
| } : null, | |
| { chunkTimeoutMs } | |
| ); | |
| const finalLoaded = shardLoaded[shardIndex] > 0 ? shardLoaded[shardIndex] : shardTotals[shardIndex] > 0 ? shardTotals[shardIndex] : headerTotal; | |
| shardLoaded[shardIndex] = finalLoaded; | |
| if (finalLoaded > 0 && shardTotals[shardIndex] < finalLoaded) { | |
| shardTotals[shardIndex] = finalLoaded; | |
| } | |
| emitAggregateProgress(); | |
| if (putPromise) { | |
| const putResult = await putPromise; | |
| if (putResult.ok) { | |
| this._modelCacheState = "stored"; | |
| this._runtimeNotes.push("model_cache_stored"); | |
| } else { | |
| const putErrorText = String(putResult.error || "").toLowerCase(); | |
| if (putErrorText.includes("abort") || putErrorText.includes("cancel")) { | |
| throw putResult.error || new Error("Model cache prefetch was aborted."); | |
| } | |
| this._modelCacheState = "store_failed"; | |
| this._runtimeNotes.push("model_cache_store_failed"); | |
| throw new Error("Failed to store prefetched model in browser cache."); | |
| } | |
| } | |
| } | |
| if (modelUrls.length > 1) { | |
| this._runtimeNotes.push("model_split_cache_prefetched"); | |
| } | |
| return 1; | |
| } catch (error) { | |
| this._modelCacheState = "error"; | |
| const text = String(error || "").toLowerCase(); | |
| if (text.includes("abort")) { | |
| this._runtimeNotes.push("model_cache_prefetch_aborted"); | |
| } else if (text.includes("quota") || text.includes("storage")) { | |
| this._runtimeNotes.push("model_cache_quota_exceeded"); | |
| } else { | |
| this._runtimeNotes.push("model_cache_error"); | |
| } | |
| throw error; | |
| } finally { | |
| this._clearTransferAbortController(controller); | |
| } | |
| } | |
| async evictModelFromCache(url, options = {}) { | |
| this._modelCacheName = this._resolveCacheName(options); | |
| const modelUrls = expandModelShardUrls(url); | |
| if (modelUrls.length === 0) { | |
| return false; | |
| } | |
| if (!globalThis.caches || typeof globalThis.caches.open !== "function") { | |
| this._modelCacheState = "unavailable"; | |
| return false; | |
| } | |
| try { | |
| const cache = await globalThis.caches.open(this._modelCacheName); | |
| let removedCount = 0; | |
| for (const modelUrl of modelUrls) { | |
| const cacheKey = normalizeAbsoluteUrl(modelUrl); | |
| const removed = await cache.delete(cacheKey); | |
| if (removed) { | |
| removedCount += 1; | |
| } | |
| } | |
| const removedAny = removedCount > 0; | |
| this._modelCacheState = removedAny ? "evicted" : "miss"; | |
| if (removedAny) { | |
| this._runtimeNotes.push("model_cache_evicted"); | |
| } | |
| if (modelUrls.length > 1) { | |
| this._runtimeNotes.push(`model_split_cache_evicted:${removedCount}/${modelUrls.length}`); | |
| } | |
| return removedAny; | |
| } catch (_) { | |
| this._modelCacheState = "error"; | |
| this._runtimeNotes.push("model_cache_error"); | |
| return false; | |
| } | |
| } | |
| async _ensureCore() { | |
| if (this._core) { | |
| this._applyCoreLogLevel(); | |
| return this._core; | |
| } | |
| const candidates = []; | |
| if (this._config.coreModuleFactory) { | |
| candidates.push({ | |
| variant: "custom", | |
| factoryPromise: Promise.resolve(this._config.coreModuleFactory), | |
| wasmUrl: this._config.wasmUrl | |
| }); | |
| } else { | |
| if (this._preferMemory64 && typeof this._config.coreModuleUrlMem64 === "string" && this._config.coreModuleUrlMem64.length > 0) { | |
| candidates.push({ | |
| variant: "wasm64", | |
| factoryPromise: importCoreFactory(this._config.coreModuleUrlMem64), | |
| wasmUrl: this._config.wasmUrlMem64 || this._config.wasmUrl | |
| }); | |
| } | |
| candidates.push({ | |
| variant: "wasm32", | |
| factoryPromise: importCoreFactory(this._config.coreModuleUrl ?? "./llama_webgpu_core.js"), | |
| wasmUrl: this._config.wasmUrl | |
| }); | |
| } | |
| let lastError = null; | |
| for (const candidate of candidates) { | |
| if (candidate.variant === "wasm64") { | |
| this._runtimeNotes.push("core_mem64_attempt"); | |
| } | |
| try { | |
| const moduleFactory = await candidate.factoryPromise; | |
| const initTimeoutMs = this._resolveCoreInitTimeoutMs(); | |
| this._core = await Promise.race([ | |
| moduleFactory({ | |
| locateFile: (path, prefix) => { | |
| if (path.endsWith(".wasm") && candidate.wasmUrl) { | |
| return candidate.wasmUrl; | |
| } | |
| return `${prefix}${path}`; | |
| }, | |
| print: (msg) => { | |
| this._emitLogger("log", msg); | |
| }, | |
| printErr: (msg) => { | |
| const text = String(msg ?? ""); | |
| this._lastCoreErrorText = text; | |
| const trimmed = text.trim(); | |
| if (trimmed.length > 0) { | |
| const loweredTrimmed = trimmed.toLowerCase(); | |
| const isGenericAbort = loweredTrimmed === "aborted(native code called abort())" || loweredTrimmed === "native code called abort()" || loweredTrimmed === "aborted"; | |
| if (!isGenericAbort) { | |
| this._lastCoreErrorHint = trimmed; | |
| } | |
| } | |
| const classification = this._classifyCoreErrorLine(text); | |
| if (classification === "ignore") { | |
| return; | |
| } | |
| if (classification === "warmup") { | |
| if (this._logLevel >= 2) { | |
| this._suppressedWarmupWarningCount += 1; | |
| if (!this._didReportWarmupWarningSuppression) { | |
| this._didReportWarmupWarningSuppression = true; | |
| if (this._logLevel <= 2) { | |
| this._emitLogger( | |
| "log", | |
| "info: suppressing verbose warmup op logs; set bridge/runtime log level to Debug to inspect all warmup details." | |
| ); | |
| } | |
| } | |
| this._pushRuntimeNote("warmup_warning_suppressed"); | |
| return; | |
| } | |
| this._emitLogger("log", trimmed.length > 0 ? trimmed : text); | |
| return; | |
| } | |
| if (classification === "warn") { | |
| this._emitLogger("warn", trimmed.length > 0 ? trimmed : text); | |
| return; | |
| } | |
| if (classification === "info") { | |
| this._emitLogger("log", trimmed.length > 0 ? trimmed : text); | |
| return; | |
| } | |
| this._emitLogger("error", trimmed.length > 0 ? trimmed : text); | |
| }, | |
| onAbort: (reason) => { | |
| const text = String(reason ?? "").trim(); | |
| if (text.length > 0) { | |
| const token = text.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, ""); | |
| if (token.length > 0) { | |
| this._runtimeNotes.push(`core_abort:${token}`); | |
| } | |
| this._emitLogger("error", `core abort: ${text}`); | |
| } else { | |
| this._runtimeNotes.push("core_abort"); | |
| this._emitLogger("error", "core abort"); | |
| } | |
| } | |
| }), | |
| new Promise((_, reject) => { | |
| globalThis.setTimeout(() => { | |
| reject(new Error(`Bridge core init timeout (${initTimeoutMs}ms)`)); | |
| }, initTimeoutMs); | |
| }) | |
| ]); | |
| this._coreVariant = candidate.variant === "wasm64" ? "wasm64" : "wasm32"; | |
| if (candidate.variant === "wasm64") { | |
| this._runtimeNotes.push("core_mem64_active"); | |
| } else if (candidate.variant === "wasm32") { | |
| this._runtimeNotes.push("core_wasm32_active"); | |
| } | |
| break; | |
| } catch (error) { | |
| lastError = error; | |
| if (candidate.variant === "wasm64") { | |
| this._runtimeNotes.push("core_mem64_unavailable"); | |
| continue; | |
| } | |
| throw error; | |
| } | |
| } | |
| if (!this._core) { | |
| throw lastError || new Error("Failed to initialize bridge core module"); | |
| } | |
| this._applyCoreLogLevel(); | |
| return this._core; | |
| } | |
| async _probeBackends() { | |
| try { | |
| const core = await this._ensureCore(); | |
| const probeResult = Number( | |
| await core.ccall("llamadart_webgpu_probe", "number", [], [], { async: true }) | |
| ); | |
| const json = core.ccall("llamadart_webgpu_backends_json", "string", [], []); | |
| let parsed = []; | |
| try { | |
| parsed = JSON.parse(json || "[]"); | |
| } catch (_) { | |
| parsed = []; | |
| } | |
| this._backendLabels = Array.isArray(parsed) ? parsed.map((v) => String(v)) : []; | |
| this._gpuActive = probeResult === 1; | |
| } catch (_err) { | |
| this._backendLabels = []; | |
| this._gpuActive = false; | |
| } | |
| return this._gpuActive; | |
| } | |
| async loadModelFromUrl(url, options = {}, resolvedLoadState = null) { | |
| const operationSignal = options?.signal || null; | |
| const abortMessage = "Model load was cancelled."; | |
| throwIfAborted(operationSignal, abortMessage); | |
| this._abortRequested = false; | |
| this._runtimeNotes = []; | |
| this._suppressedWarmupWarningCount = 0; | |
| this._didReportWarmupWarningSuppression = false; | |
| await this._probeBackends(); | |
| throwIfAborted(operationSignal, abortMessage); | |
| const core = await this._ensureCore(); | |
| throwIfAborted(operationSignal, abortMessage); | |
| const modelUrls = expandModelShardUrls(url); | |
| if (modelUrls.length === 0) { | |
| throw new Error("Model URL is empty."); | |
| } | |
| this._releaseLoadedModel(core); | |
| this._mmProjSourceUrl = null; | |
| const configuredPoolHint = Number(this._threadPoolSizeHint); | |
| this._syncThreadPoolSizeHintFromCore(); | |
| const coreSupportsPthreads = this._coreSupportsPthreads(); | |
| this._pushRuntimeNote(`core_pthreads:${coreSupportsPthreads ? 1 : 0}`); | |
| if (!coreSupportsPthreads && Number.isFinite(configuredPoolHint) && configuredPoolHint > 1) { | |
| this._runtimeNotes.push("threads_capped_no_pthread"); | |
| } | |
| this._nCtx = Number(options.nCtx) > 0 ? Number(options.nCtx) : this._nCtx; | |
| const requestedThreads = Number(options.nThreads); | |
| if (Number.isFinite(requestedThreads) && requestedThreads > 0) { | |
| this._threads = this._capThreadsToPool(requestedThreads); | |
| } else { | |
| this._threads = this._capThreadsToPool(this._resolveAutoThreadCount()); | |
| } | |
| const requestedThreadsBatch = Number(options.nThreadsBatch); | |
| if (Number.isFinite(requestedThreadsBatch) && requestedThreadsBatch > 0) { | |
| this._threadsBatch = this._capThreadsToPool( | |
| requestedThreadsBatch, | |
| { noteTag: "threads_batch_capped_pool" } | |
| ); | |
| } else { | |
| this._threadsBatch = this._threads; | |
| } | |
| const requestedGpuLayers = Number(options.nGpuLayers); | |
| if (Number.isFinite(requestedGpuLayers)) { | |
| this._nGpuLayers = Math.trunc(requestedGpuLayers); | |
| } | |
| const isCpuModelMode = this._nGpuLayers === 0; | |
| const requestedBatch = Number(options.nBatch); | |
| this._nBatch = Number.isFinite(requestedBatch) && requestedBatch > 0 ? Math.max(32, Math.trunc(requestedBatch)) : isCpuModelMode ? Math.min(this._nCtx, 512) : 0; | |
| const requestedUbatch = Number(options.nUbatch); | |
| this._nUbatch = Number.isFinite(requestedUbatch) && requestedUbatch > 0 ? Math.max(32, Math.trunc(requestedUbatch)) : isCpuModelMode ? Math.min(this._nBatch || 256, 256) : 0; | |
| if (this._nBatch > 0 && this._nBatch > this._nCtx) { | |
| this._nBatch = this._nCtx; | |
| } | |
| if (this._nUbatch > 0 && this._nBatch > 0 && this._nUbatch > this._nBatch) { | |
| this._nUbatch = this._nBatch; | |
| } | |
| if (resolvedLoadState) { | |
| Object.assign(this, resolvedLoadState); | |
| this._threads = this._capThreadsToPool(this._threads); | |
| this._threadsBatch = this._capThreadsToPool(this._threadsBatch, { noteTag: "threads_batch_capped_pool" }); | |
| } else { | |
| this._resolveNativeLoadOptions(options); | |
| } | |
| if (Number.isFinite(this._threadPoolSizeHint) && this._threadPoolSizeHint > 0) { | |
| this._pushRuntimeNote(`thread_pool_size:${this._threadPoolSizeHint}`); | |
| } | |
| if (!isCrossOriginIsolatedRuntime()) { | |
| this._runtimeNotes.push("threads_capped_no_coi"); | |
| if (this._threads > 1) { | |
| this._threads = 1; | |
| } | |
| if (this._threadsBatch > 1) { | |
| this._threadsBatch = 1; | |
| } | |
| } | |
| this._pushRuntimeNote(`threads_batch:${this._threadsBatch}`); | |
| if (this._nBatch > 0) { | |
| this._pushRuntimeNote(`n_batch:${this._nBatch}`); | |
| } | |
| if (this._nUbatch > 0) { | |
| this._pushRuntimeNote(`n_ubatch:${this._nUbatch}`); | |
| } | |
| if (this._nSeqMax > 0) { | |
| this._pushRuntimeNote(`n_seq_max:${this._nSeqMax}`); | |
| } | |
| if (!resolvedLoadState && isCpuModelMode && !Number.isFinite(requestedBatch) && !Number.isFinite(requestedUbatch)) { | |
| this._runtimeNotes.push("cpu_batch_tuned_default"); | |
| } | |
| if (this._isSafari && this._nGpuLayers > 0) { | |
| const requestedSafariMaxLayers = Number(options.safariMaxGpuLayers); | |
| const safariMaxGpuLayers = Number.isFinite(requestedSafariMaxLayers) ? Math.max(1, Math.trunc(requestedSafariMaxLayers)) : 1; | |
| if (this._nGpuLayers > safariMaxGpuLayers) { | |
| this._nGpuLayers = safariMaxGpuLayers; | |
| this._runtimeNotes.push(`safari_gpu_layers_capped:${safariMaxGpuLayers}`); | |
| } | |
| } | |
| this._loadedModelUrl = cloneModelSource(url); | |
| if (modelUrls.length > 1) { | |
| this._runtimeNotes.push(`model_split_detected:${modelUrls.length}`); | |
| } | |
| let loadedViaRemoteFetch = false; | |
| let remoteFetchReloadUrl = null; | |
| const remoteFetchReloadChunkBytes = this._resolveRemoteFetchChunkBytes(options); | |
| if (modelUrls.length === 1) { | |
| const remoteResult = await this._tryLoadModelFromRemoteFetchBackend( | |
| core, | |
| modelUrls[0], | |
| options | |
| ); | |
| loadedViaRemoteFetch = remoteResult.loaded === true; | |
| if (loadedViaRemoteFetch) { | |
| remoteFetchReloadUrl = modelUrls[0]; | |
| } | |
| } else { | |
| this._runtimeNotes.push("model_fetch_backend_skipped_split"); | |
| } | |
| if (!loadedViaRemoteFetch) { | |
| ensureFsDirectory(core.FS, "/models"); | |
| const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null; | |
| const shardLoaded = new Array(modelUrls.length).fill(0); | |
| const shardTotals = new Array(modelUrls.length).fill(0); | |
| const emitAggregateProgress = () => { | |
| if (!progressCallback) { | |
| return; | |
| } | |
| const loaded = sumProgressValues(shardLoaded); | |
| const total = sumProgressValues(shardTotals); | |
| progressCallback({ | |
| loaded, | |
| total: total > 0 ? total : loaded | |
| }); | |
| }; | |
| const modelPaths = []; | |
| let totalModelBytes = 0; | |
| const maxStreamResumeRetries = Number.isFinite(options.streamResumeRetries) ? Math.max(0, Math.trunc(options.streamResumeRetries)) : 8; | |
| const streamChunkTimeoutMs = this._resolveStreamChunkTimeoutMs( | |
| options, | |
| 9e4 | |
| ); | |
| try { | |
| for (let shardIndex = 0; shardIndex < modelUrls.length; shardIndex += 1) { | |
| const shardUrl = modelUrls[shardIndex]; | |
| const fileName = basenameFromUrl(shardUrl); | |
| const modelPath = `/models/${fileName}`; | |
| modelPaths.push(modelPath); | |
| let shardBytes = 0; | |
| let resumeOffset = 0; | |
| let resumeAttempt = 0; | |
| let activeShardUrl = shardUrl; | |
| let knownTotalBytes = 0; | |
| while (true) { | |
| throwIfAborted(operationSignal, abortMessage); | |
| const requestHeaders = resumeOffset > 0 ? { Range: `bytes=${resumeOffset}-` } : null; | |
| const response = await this._getCachedModelResponse(activeShardUrl, { | |
| ...options, | |
| requireReadableStream: true, | |
| useCache: requestHeaders ? false : options.useCache, | |
| force: requestHeaders ? true : options.force, | |
| requestHeaders | |
| }); | |
| if (!response.ok) { | |
| throw new Error( | |
| `Failed to fetch model shard: ${response.status} ${response.statusText}` | |
| ); | |
| } | |
| const responseUrl = typeof response.url === "string" && response.url.length > 0 ? response.url : activeShardUrl; | |
| if (responseUrl !== activeShardUrl) { | |
| activeShardUrl = responseUrl; | |
| if (resumeOffset > 0) { | |
| this._runtimeNotes.push("model_stream_resume_redirect"); | |
| } | |
| } | |
| if (resumeOffset > 0 && response.status !== 206) { | |
| this._runtimeNotes.push(`model_stream_resume_status:${response.status}`); | |
| throw new Error( | |
| `Range resume not honored for model shard: ${response.status} ${response.statusText}` | |
| ); | |
| } | |
| if (!hasReadableResponseStream(response)) { | |
| this._runtimeNotes.push("model_response_nostream"); | |
| const declaredBytes = inferResponseTotalBytes(response, 0); | |
| const declaredText = declaredBytes > 0 ? `${declaredBytes} bytes` : "unknown size"; | |
| throw new Error( | |
| `Model response did not expose a readable stream (${declaredText}).` | |
| ); | |
| } | |
| this._runtimeNotes.push("model_response_stream"); | |
| const responseTotal = inferResponseTotalBytes(response, knownTotalBytes); | |
| if (responseTotal > 0) { | |
| knownTotalBytes = responseTotal; | |
| shardTotals[shardIndex] = responseTotal; | |
| } | |
| try { | |
| shardBytes = await writeResponseToFsFileWithProgress( | |
| response, | |
| core.FS, | |
| modelPath, | |
| progressCallback ? (progress) => { | |
| const loaded = Number(progress?.loaded) || 0; | |
| const total = Number(progress?.total) || 0; | |
| shardLoaded[shardIndex] = loaded; | |
| if (total > 0) { | |
| shardTotals[shardIndex] = total; | |
| } | |
| emitAggregateProgress(); | |
| } : null, | |
| { | |
| useBigIntPosition: this._coreVariant === "wasm64", | |
| startOffset: resumeOffset, | |
| preservePartialOnError: true, | |
| totalBytes: knownTotalBytes, | |
| chunkTimeoutMs: streamChunkTimeoutMs, | |
| signal: operationSignal, | |
| abortMessage | |
| } | |
| ); | |
| break; | |
| } catch (error) { | |
| const text = String(error || "").toLowerCase(); | |
| const loadedBytes = Number( | |
| error?.llamadartLoadedBytes | |
| ); | |
| if (Number.isFinite(loadedBytes) && loadedBytes >= 0) { | |
| const normalizedLoaded = Math.trunc(loadedBytes); | |
| this._runtimeNotes.push(`model_fs_write_loaded:${normalizedLoaded}`); | |
| if (normalizedLoaded > resumeOffset) { | |
| resumeOffset = normalizedLoaded; | |
| shardLoaded[shardIndex] = normalizedLoaded; | |
| if (knownTotalBytes > 0 && shardTotals[shardIndex] < knownTotalBytes) { | |
| shardTotals[shardIndex] = knownTotalBytes; | |
| } | |
| emitAggregateProgress(); | |
| } | |
| } | |
| const shouldRetryResume = isRetryableStreamNetworkError(error) && resumeOffset > 0 && resumeAttempt < maxStreamResumeRetries; | |
| if (shouldRetryResume) { | |
| resumeAttempt += 1; | |
| this._runtimeNotes.push(`model_stream_resume_retry:${resumeAttempt}`); | |
| this._runtimeNotes.push(`model_stream_resume_offset:${resumeOffset}`); | |
| continue; | |
| } | |
| if (text.includes("bigint")) { | |
| this._runtimeNotes.push("model_fs_write_bigint_error"); | |
| } else if (text.includes("abort")) { | |
| this._runtimeNotes.push("model_fs_write_abort"); | |
| } else if (text.includes("array buffer allocation failed")) { | |
| this._runtimeNotes.push("model_fs_write_arraybuffer_oom"); | |
| } else if (isRetryableStreamNetworkError(error)) { | |
| this._runtimeNotes.push("model_fs_write_network_error"); | |
| this._runtimeNotes.push("model_fs_write_failed"); | |
| } else { | |
| this._runtimeNotes.push("model_fs_write_failed"); | |
| } | |
| throw error; | |
| } | |
| } | |
| shardLoaded[shardIndex] = shardBytes; | |
| if (shardTotals[shardIndex] < shardBytes) { | |
| shardTotals[shardIndex] = shardBytes; | |
| } | |
| totalModelBytes += shardBytes; | |
| emitAggregateProgress(); | |
| } | |
| this._modelPaths = modelPaths; | |
| this._modelPath = modelPaths[0] || null; | |
| this._modelBytes = totalModelBytes; | |
| let rc = 0; | |
| let nativeLoadStarted = false; | |
| let nativeAbortCleanupDone = false; | |
| try { | |
| throwIfAborted(operationSignal, abortMessage); | |
| nativeLoadStarted = true; | |
| rc = Number( | |
| await core.ccall( | |
| "llamadart_webgpu_load_model", | |
| "number", | |
| [ | |
| "string", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| ...this._nativeLoadOptionTypes() | |
| ], | |
| [ | |
| this._modelPath, | |
| this._nCtx, | |
| this._threads, | |
| this._threadsBatch, | |
| this._nBatch, | |
| this._nUbatch, | |
| this._nGpuLayers, | |
| ...this._nativeLoadOptionValues() | |
| ], | |
| { async: true } | |
| ) | |
| ); | |
| if (operationSignal?.aborted) { | |
| try { | |
| core.ccall("llamadart_webgpu_shutdown", null, [], []); | |
| nativeAbortCleanupDone = true; | |
| } catch (_) { | |
| } | |
| throw createAbortError(abortMessage); | |
| } | |
| } catch (error) { | |
| if (operationSignal?.aborted && nativeLoadStarted && !nativeAbortCleanupDone) { | |
| try { | |
| core.ccall("llamadart_webgpu_shutdown", null, [], []); | |
| nativeAbortCleanupDone = true; | |
| } catch (_) { | |
| } | |
| error = createAbortError(abortMessage); | |
| } | |
| const text = String(error || "").toLowerCase(); | |
| if (text.includes("bigint")) { | |
| this._runtimeNotes.push("model_load_ccall_bigint_error"); | |
| } else if (text.includes("abort")) { | |
| this._runtimeNotes.push("model_load_ccall_abort"); | |
| } else { | |
| this._runtimeNotes.push("model_load_ccall_failed"); | |
| } | |
| throw error; | |
| } | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to load GGUF model", rc)); | |
| } | |
| if (modelUrls.length > 1) { | |
| this._runtimeNotes.push(`model_split_loaded:${modelUrls.length}`); | |
| } | |
| } catch (error) { | |
| this._modelBytes = 0; | |
| this._modelPath = null; | |
| this._modelPaths = []; | |
| if (operationSignal?.aborted) { | |
| this._loadedModelUrl = null; | |
| } | |
| for (const modelPath of modelPaths) { | |
| this._deleteFsFile(modelPath); | |
| } | |
| throw error; | |
| } | |
| } | |
| const shouldProbeSafariGpu = this._isSafari && this._nGpuLayers > 0 && options.safariGpuProbe !== false; | |
| if (shouldProbeSafariGpu) { | |
| if (loadedViaRemoteFetch) { | |
| this._runtimeNotes.push("safari_probe_on_fetch_backend"); | |
| } | |
| const defaultProbePrompts = [ | |
| "user: hi\nassistant:", | |
| "user: say hello in one short sentence\nassistant:" | |
| ]; | |
| const probePrompts = Array.isArray(options.safariProbePrompts) ? options.safariProbePrompts.map((v) => String(v || "").trim()).filter((v) => v.length > 0) : typeof options.safariProbePrompt === "string" && options.safariProbePrompt.trim().length > 0 ? [options.safariProbePrompt.trim()] : defaultProbePrompts; | |
| const probeTokensRaw = Number(options.safariProbeTokens); | |
| const probeTokens = Number.isFinite(probeTokensRaw) && probeTokensRaw > 0 ? Math.min(Math.trunc(probeTokensRaw), 96) : 48; | |
| const runProbe = async (probePrompt, probeSeed) => { | |
| try { | |
| const probeOutput = await this.createCompletion(probePrompt, { | |
| nPredict: probeTokens, | |
| temp: 0, | |
| topK: 1, | |
| topP: 1, | |
| penalty: 1, | |
| seed: probeSeed | |
| }); | |
| return !looksLikeCorruptedGeneration(probeOutput); | |
| } catch (_) { | |
| return false; | |
| } | |
| }; | |
| let initialProbePassed = true; | |
| for (let i = 0; i < probePrompts.length; i += 1) { | |
| const ok = await runProbe(probePrompts[i], i + 1); | |
| if (!ok) { | |
| initialProbePassed = false; | |
| break; | |
| } | |
| } | |
| if (!initialProbePassed) { | |
| this._runtimeNotes.push("safari_gpu_probe_failed"); | |
| const retryCandidates = []; | |
| if (this._nGpuLayers > 1) { | |
| retryCandidates.push(1); | |
| } | |
| retryCandidates.push(0); | |
| let stabilized = false; | |
| for (const candidateLayers of retryCandidates) { | |
| try { | |
| core.ccall("llamadart_webgpu_shutdown", null, [], []); | |
| } catch (_) { | |
| } | |
| let retryRc = 0; | |
| if (loadedViaRemoteFetch) { | |
| const reloadUrl = remoteFetchReloadUrl || modelUrls[0] || null; | |
| if (!reloadUrl) { | |
| continue; | |
| } | |
| this._runtimeNotes.push(`safari_probe_remote_fetch_retry:${candidateLayers}`); | |
| retryRc = Number( | |
| await core.ccall( | |
| "llamadart_webgpu_load_model_from_url", | |
| "number", | |
| [ | |
| "string", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| ...this._nativeLoadOptionTypes() | |
| ], | |
| [ | |
| reloadUrl, | |
| this._nCtx, | |
| this._threads, | |
| this._threadsBatch, | |
| this._nBatch, | |
| this._nUbatch, | |
| candidateLayers, | |
| remoteFetchReloadChunkBytes, | |
| ...this._nativeLoadOptionValues() | |
| ], | |
| { async: true } | |
| ) | |
| ); | |
| } else { | |
| retryRc = Number( | |
| await core.ccall( | |
| "llamadart_webgpu_load_model", | |
| "number", | |
| [ | |
| "string", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| ...this._nativeLoadOptionTypes() | |
| ], | |
| [ | |
| this._modelPath, | |
| this._nCtx, | |
| this._threads, | |
| this._threadsBatch, | |
| this._nBatch, | |
| this._nUbatch, | |
| candidateLayers, | |
| ...this._nativeLoadOptionValues() | |
| ], | |
| { async: true } | |
| ) | |
| ); | |
| } | |
| if (retryRc !== 0) { | |
| continue; | |
| } | |
| this._nGpuLayers = candidateLayers; | |
| if (candidateLayers === 0) { | |
| this._runtimeNotes.push("safari_fallback_cpu"); | |
| stabilized = true; | |
| break; | |
| } | |
| let retryProbePassed = true; | |
| for (let i = 0; i < probePrompts.length; i += 1) { | |
| const ok = await runProbe(probePrompts[i], i + 11); | |
| if (!ok) { | |
| retryProbePassed = false; | |
| break; | |
| } | |
| } | |
| if (retryProbePassed) { | |
| this._runtimeNotes.push(`safari_gpu_layers_capped:${candidateLayers}`); | |
| stabilized = true; | |
| break; | |
| } | |
| } | |
| if (!stabilized) { | |
| throw new Error("Safari GPU probe failed and fallback attempts were unsuccessful."); | |
| } | |
| } else { | |
| this._runtimeNotes.push("safari_gpu_probe_passed"); | |
| } | |
| } | |
| try { | |
| const effectiveNctx = Number(core.ccall("llamadart_webgpu_get_context_size", "number", [], [])); | |
| if (effectiveNctx > 0) { | |
| this._nCtx = effectiveNctx; | |
| } | |
| } catch (_) { | |
| } | |
| this._mmProjPath = null; | |
| this._mmSupportsVision = false; | |
| this._mmSupportsAudio = false; | |
| this._mediaFileCounter = 0; | |
| this._stateFileCounter = 0; | |
| this._stagedMediaPaths = []; | |
| this._gpuActive = this._gpuActive && this._nGpuLayers > 0; | |
| if (this._releaseModelFiles() > 0) { | |
| this._runtimeNotes.push("model_file_released"); | |
| } | |
| this._emitSuppressedWarmupWarningSummaryIfNeeded(); | |
| return 1; | |
| } | |
| async loadMultimodalProjector(url) { | |
| if (!this._core || this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| if (typeof url !== "string" || url.length === 0) { | |
| throw new Error("Multimodal projector URL/path is empty."); | |
| } | |
| const core = await this._ensureCore(); | |
| ensureFsDirectory(core.FS, "/mmproj"); | |
| const fileName = basenameFromUrl(url); | |
| const mmprojPath = `/mmproj/${fileName}`; | |
| const fetchTimeoutMs = this._resolveFetchTimeoutMs({}, 18e4); | |
| const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs({}, 9e4); | |
| let lastError = null; | |
| for (let attempt = 0; attempt < 2; attempt += 1) { | |
| try { | |
| const response = await this._fetchWithTimeout( | |
| url, | |
| { cache: "no-store" }, | |
| fetchTimeoutMs | |
| ); | |
| if (!response.ok) { | |
| throw new Error( | |
| `Failed to fetch multimodal projector: ${response.status} ${response.statusText}` | |
| ); | |
| } | |
| this._mmProjPath = mmprojPath; | |
| await writeResponseToFsFileWithProgress( | |
| response, | |
| core.FS, | |
| this._mmProjPath, | |
| null, | |
| { | |
| useBigIntPosition: this._coreVariant === "wasm64", | |
| chunkTimeoutMs | |
| } | |
| ); | |
| const rc = Number( | |
| await core.ccall( | |
| "llamadart_webgpu_mmproj_load", | |
| "number", | |
| ["string"], | |
| [this._mmProjPath], | |
| { async: true } | |
| ) | |
| ); | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to load multimodal projector", rc)); | |
| } | |
| this._mmSupportsVision = Number( | |
| core.ccall("llamadart_webgpu_mmproj_supports_vision", "number", [], []) | |
| ) === 1; | |
| this._mmSupportsAudio = Number( | |
| core.ccall("llamadart_webgpu_mmproj_supports_audio", "number", [], []) | |
| ) === 1; | |
| this._mmProjSourceUrl = url; | |
| return 1; | |
| } catch (error) { | |
| lastError = error; | |
| this._mmProjPath = null; | |
| this._mmSupportsVision = false; | |
| this._mmSupportsAudio = false; | |
| this._deleteFsFile(mmprojPath); | |
| const retryable = isRetryableStreamNetworkError(error); | |
| if (!retryable || attempt >= 1) { | |
| throw error; | |
| } | |
| this._runtimeNotes.push(`mmproj_load_retry:${attempt + 1}`); | |
| } | |
| } | |
| if (lastError) { | |
| throw lastError; | |
| } | |
| throw new Error("Failed to load multimodal projector"); | |
| } | |
| async unloadMultimodalProjector() { | |
| if (!this._core) { | |
| this._mmProjPath = null; | |
| this._mmSupportsVision = false; | |
| this._mmSupportsAudio = false; | |
| this._mmProjSourceUrl = null; | |
| return; | |
| } | |
| const mmprojPath = this._mmProjPath; | |
| const rc = Number( | |
| this._core.ccall("llamadart_webgpu_mmproj_free", "number", [], []) | |
| ); | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to unload multimodal projector", rc)); | |
| } | |
| this._deleteFsFile(mmprojPath); | |
| this._mmProjPath = null; | |
| this._mmSupportsVision = false; | |
| this._mmSupportsAudio = false; | |
| this._mmProjSourceUrl = null; | |
| } | |
| // Bytes the core heap can still allocate, or null for a core without the | |
| // probe. | |
| _heapHeadroomBytes(core) { | |
| try { | |
| const value = Number(core.ccall("llamadart_webgpu_heap_headroom_bytes", "number", [], [])); | |
| return Number.isFinite(value) && value >= 0 ? value : null; | |
| } catch (_) { | |
| return null; | |
| } | |
| } | |
| // Throws when `bytes` more would not fit in the core heap, naming the | |
| // memory64 core when the wasm32 core's 4 GiB limit is the reason. | |
| _requireHeapHeadroom(core, bytes, what) { | |
| const headroom = this._heapHeadroomBytes(core); | |
| if (headroom == null || !(bytes > headroom)) { | |
| return; | |
| } | |
| const mib = (value, round) => `${round(value / (1024 * 1024))} MiB`; | |
| const detail = this._coreVariant === "wasm32" ? "the wasm32 core addresses at most 4 GiB; memory64 is required: load the bridge with coreModuleUrlMem64 and preferMemory64 in a browser with WebAssembly memory64 support" : "free memory by unloading other models or use a smaller draft model"; | |
| throw new Error( | |
| `${what} needs about ${mib(bytes, Math.ceil)} but only ${mib(headroom, Math.floor)} of WebAssembly memory is left; ${detail}.` | |
| ); | |
| } | |
| async loadDraftModel(url, options = {}) { | |
| if (!this._core || this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| if (typeof url !== "string" || url.length === 0) { | |
| throw new Error("Draft model URL is empty."); | |
| } | |
| const core = this._core; | |
| const abortMessage = "Draft model load was cancelled."; | |
| throwIfAborted(options.signal || null, abortMessage); | |
| const rc = Number(await core.ccall("llamadart_webgpu_draft_model_free", "number", [], [], { async: true })); | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to release the draft model", rc)); | |
| } | |
| this._draftModel = null; | |
| ensureFsDirectory(core.FS, "/draft"); | |
| const draftPath = `/draft/${basenameFromUrl(url)}`; | |
| const controller = this._beginTransferAbortController(); | |
| const onCallerAbort = () => controller?.abort(); | |
| options.signal?.addEventListener("abort", onCallerAbort, { once: true }); | |
| const signal = controller?.signal ?? options.signal ?? null; | |
| const targetCacheFields = [this._modelSource, this._modelCacheState, this._modelCacheName]; | |
| try { | |
| let response; | |
| try { | |
| response = await this._getCachedModelResponse(url, { | |
| useCache: options.useCache, | |
| force: options.force, | |
| signal, | |
| requireReadableStream: true | |
| }); | |
| } finally { | |
| [this._modelSource, this._modelCacheState, this._modelCacheName] = targetCacheFields; | |
| } | |
| if (!response.ok) { | |
| throw new Error(`Failed to fetch draft model: ${response.status} ${response.statusText}`); | |
| } | |
| const totalBytes = inferResponseTotalBytes(response, 0); | |
| if (totalBytes > 0) { | |
| this._requireHeapHeadroom( | |
| core, | |
| this._nGpuLayers === 0 ? totalBytes * 2 : totalBytes, | |
| "The draft model" | |
| ); | |
| } | |
| const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null; | |
| await writeResponseToFsFileWithProgress( | |
| response, | |
| core.FS, | |
| draftPath, | |
| progressCallback, | |
| { | |
| useBigIntPosition: this._coreVariant === "wasm64", | |
| totalBytes, | |
| chunkTimeoutMs: this._resolveStreamChunkTimeoutMs({}, 9e4), | |
| signal, | |
| abortMessage | |
| } | |
| ); | |
| throwIfAborted(signal, abortMessage); | |
| const loadRc = Number( | |
| await core.ccall( | |
| "llamadart_webgpu_draft_model_load", | |
| "number", | |
| ["string"], | |
| [draftPath], | |
| { async: true } | |
| ) | |
| ); | |
| if (loadRc !== 0) { | |
| let message = this._coreErrorMessage("Failed to load draft model", loadRc); | |
| if (this._coreVariant === "wasm32" && /memory|alloc/i.test(message)) { | |
| message += "; the wasm32 core addresses at most 4 GiB, so a larger draft model needs the memory64 core"; | |
| } | |
| throw new Error(message); | |
| } | |
| const info = JSON.parse( | |
| core.ccall("llamadart_webgpu_draft_model_info_json", "string", [], []) || "{}" | |
| ); | |
| this._draftModel = { | |
| url, | |
| options: { useCache: options.useCache, force: false }, | |
| info | |
| }; | |
| return { ...info }; | |
| } finally { | |
| options.signal?.removeEventListener("abort", onCallerAbort); | |
| this._clearTransferAbortController(controller); | |
| this._deleteFsFile(draftPath); | |
| } | |
| } | |
| async unloadDraftModel() { | |
| if (!this._core) { | |
| this._draftModel = null; | |
| return; | |
| } | |
| const rc = Number( | |
| await this._core.ccall("llamadart_webgpu_draft_model_free", "number", [], [], { async: true }) | |
| ); | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to unload the draft model", rc)); | |
| } | |
| this._draftModel = null; | |
| } | |
| // Writes one n-gram cache source to the core filesystem and returns its | |
| // path; the caller deletes it. | |
| async _stageNgramCache(source, name) { | |
| const core = this._core; | |
| ensureFsDirectory(core.FS, "/speculative"); | |
| this._speculativeFileCounter += 1; | |
| const path = `/speculative/${name}_${Date.now()}_${this._speculativeFileCounter}.lcs`; | |
| if (typeof source === "string") { | |
| const response = await this._fetchWithTimeout( | |
| source, | |
| { cache: "no-store" }, | |
| this._resolveFetchTimeoutMs({}, 18e4) | |
| ); | |
| if (!response.ok) { | |
| throw new Error(`Failed to fetch the ${name} n-gram cache: ${response.status} ${response.statusText}`); | |
| } | |
| core.FS.writeFile(path, new Uint8Array(await response.arrayBuffer())); | |
| } else { | |
| const bytes = source instanceof ArrayBuffer ? new Uint8Array(source) : new Uint8Array(source.buffer, source.byteOffset, source.byteLength); | |
| core.FS.writeFile(path, bytes); | |
| } | |
| return path; | |
| } | |
| // Passes the speculative settings to the core for the next | |
| // begin_generation, which consumes them. Returns the staged cache files. | |
| async _setNextSpeculative(speculative, maxTokens, stagedPaths) { | |
| const core = this._core; | |
| const staticPath = speculative.ngramCacheStatic == null ? "" : await this._stageNgramCache(speculative.ngramCacheStatic, "static"); | |
| if (staticPath) { | |
| stagedPaths.push(staticPath); | |
| } | |
| const dynamicPath = speculative.ngramCacheDynamic == null ? "" : await this._stageNgramCache(speculative.ngramCacheDynamic, "dynamic"); | |
| if (dynamicPath) { | |
| stagedPaths.push(dynamicPath); | |
| } | |
| const rc = Number(core.ccall( | |
| "llamadart_webgpu_set_next_speculative", | |
| "number", | |
| [ | |
| "string", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "string", | |
| "string", | |
| "number" | |
| ], | |
| [ | |
| speculative.strategies.join(","), | |
| speculative.draftTokenMax, | |
| speculative.draftTokenMin, | |
| speculative.minProbability, | |
| speculative.draftSplitProbability, | |
| speculative.ngramSizeN, | |
| speculative.ngramSizeM, | |
| speculative.ngramMinHits, | |
| speculative.ngramMatch, | |
| speculative.ngramTokenMin, | |
| speculative.ngramTokenMax, | |
| staticPath, | |
| dynamicPath, | |
| maxTokens | |
| ] | |
| )); | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to configure speculative decoding", rc)); | |
| } | |
| } | |
| supportsVision() { | |
| return this._mmSupportsVision; | |
| } | |
| supportsAudio() { | |
| return this._mmSupportsAudio; | |
| } | |
| getTextToSpeechCapabilities() { | |
| if (!this._core) { | |
| return { | |
| apiVersion: 0, | |
| supported: false, | |
| modelType: 0, | |
| capabilities: 0, | |
| supportsLanguage: false, | |
| supportsSpeakerReference: false, | |
| sampleRate: 0, | |
| channels: 0, | |
| reason: "WebGPU core is not initialized" | |
| }; | |
| } | |
| const raw = this._core.ccall( | |
| "llamadart_webgpu_tts_info_json", | |
| "string", | |
| [], | |
| [] | |
| ) || "{}"; | |
| try { | |
| return JSON.parse(raw); | |
| } catch (_) { | |
| return { | |
| apiVersion: 0, | |
| supported: false, | |
| modelType: 0, | |
| capabilities: 0, | |
| supportsLanguage: false, | |
| supportsSpeakerReference: false, | |
| sampleRate: 0, | |
| channels: 0, | |
| reason: "WebGPU TTS capability response is invalid" | |
| }; | |
| } | |
| } | |
| _ensureTextToSpeechDir() { | |
| try { | |
| this._core.FS.mkdir("/tts"); | |
| } catch (_) { | |
| } | |
| } | |
| async synthesizeSpeech(options = {}) { | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| if (options.signal?.aborted) { | |
| throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); | |
| } | |
| const capabilities = this.getTextToSpeechCapabilities(); | |
| if (capabilities.supported !== true) { | |
| throw new Error( | |
| capabilities.reason || "Loaded model/projector does not support text-to-speech." | |
| ); | |
| } | |
| const text = typeof options.text === "string" ? options.text : ""; | |
| if (text.length === 0) { | |
| throw new Error("Text-to-speech input text is empty."); | |
| } | |
| if (this._textToSpeechActive) { | |
| throw new Error("Text-to-speech synthesis is already active."); | |
| } | |
| this._ensureTextToSpeechDir(); | |
| const taskId = `${Date.now()}_${++this._mediaFileCounter}`; | |
| const speakerPath = `/tts/speaker_${taskId}.bin`; | |
| const outputPath = `/tts/output_${taskId}.pcm`; | |
| const speakerBytes = toUint8Array(options.speakerAudio); | |
| if (speakerBytes && speakerBytes.length > 0) { | |
| this._core.FS.writeFile(speakerPath, speakerBytes); | |
| } | |
| const promptBatchSize = Number(options.promptBatchSize) > 0 ? Math.max(1, Math.trunc(Number(options.promptBatchSize))) : 512; | |
| const maxFrames = Number(options.maxFrames) > 0 ? Math.max(1, Math.trunc(Number(options.maxFrames))) : 512; | |
| const topK = Number.isFinite(options.topK) ? Math.max(0, Math.trunc(Number(options.topK))) : 40; | |
| const topP = Number.isFinite(options.topP) ? Number(options.topP) : 0.95; | |
| const minP = Number.isFinite(options.minP) ? Number(options.minP) : 0; | |
| const temperature = Number.isFinite(options.temperature) ? Number(options.temperature) : 0.8; | |
| const seed = Number.isInteger(options.seed) ? Number(options.seed) >>> 0 : Math.floor(Math.random() * 4294967295) >>> 0; | |
| const language = typeof options.language === "string" ? options.language : ""; | |
| let started = false; | |
| this._textToSpeechActive = true; | |
| this._textToSpeechDone = new Promise((resolve) => { | |
| this._resolveTextToSpeechDone = resolve; | |
| }); | |
| try { | |
| this._abortRequested = false; | |
| const startRc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_tts_start", | |
| "number", | |
| ["string", "string", "string", "number", "number", "number", "number", "number", "number", "number"], | |
| [ | |
| text, | |
| language, | |
| speakerBytes && speakerBytes.length > 0 ? speakerPath : null, | |
| promptBatchSize, | |
| maxFrames, | |
| topK, | |
| topP, | |
| minP, | |
| temperature, | |
| seed | |
| ], | |
| { async: true } | |
| ) | |
| ); | |
| if (startRc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to start text-to-speech synthesis", startRc)); | |
| } | |
| started = true; | |
| let progress = {}; | |
| for (; ; ) { | |
| if (this._abortRequested || options.signal?.aborted) { | |
| this.cancel(); | |
| } | |
| const stepRc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_tts_step", | |
| "number", | |
| [], | |
| [], | |
| { async: true } | |
| ) | |
| ); | |
| const rawProgress = this._core.ccall( | |
| "llamadart_webgpu_tts_progress_json", | |
| "string", | |
| [], | |
| [] | |
| ) || "{}"; | |
| progress = JSON.parse(rawProgress); | |
| options.onProgress?.(progress); | |
| if (stepRc === -6 || progress.state === 4) { | |
| throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); | |
| } | |
| if (stepRc !== 0) { | |
| throw new Error(this._coreErrorMessage("Text-to-speech synthesis failed", stepRc)); | |
| } | |
| if (progress.state === 3) { | |
| break; | |
| } | |
| await new Promise((resolve) => setTimeout(resolve, 0)); | |
| } | |
| const outputRc = Number( | |
| this._core.ccall( | |
| "llamadart_webgpu_tts_write_pcm", | |
| "number", | |
| ["string"], | |
| [outputPath] | |
| ) | |
| ); | |
| if (outputRc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to read synthesized speech", outputRc)); | |
| } | |
| const bytes = this._core.FS.readFile(outputPath); | |
| if (!(bytes instanceof Uint8Array) || bytes.byteLength === 0 || bytes.byteLength % 4 !== 0) { | |
| throw new Error("Synthesized PCM output is empty or malformed."); | |
| } | |
| const copy = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); | |
| const pcm = new Float32Array(copy); | |
| return { | |
| pcm, | |
| sampleRate: Number(capabilities.sampleRate) || 0, | |
| channels: Number(capabilities.channels) || 1, | |
| sampleCount: pcm.length, | |
| framesGenerated: Number(progress.framesGenerated) || 0, | |
| truncated: progress.truncated === true | |
| }; | |
| } finally { | |
| if (started) { | |
| try { | |
| this._core.ccall("llamadart_webgpu_tts_reset", "number", [], []); | |
| } catch (_) { | |
| } | |
| } | |
| this._deleteFsFile(speakerPath); | |
| this._deleteFsFile(outputPath); | |
| this._textToSpeechActive = false; | |
| this._resolveTextToSpeechDone?.(); | |
| this._resolveTextToSpeechDone = null; | |
| this._textToSpeechDone = null; | |
| } | |
| } | |
| getDecisionCapabilities() { | |
| const unsupported = (reason) => ({ | |
| apiVersion: DECISION_API_VERSION, | |
| supported: false, | |
| reason | |
| }); | |
| const core = this._core; | |
| if (!core) { | |
| return unsupported("WebGPU core is not initialized"); | |
| } | |
| if (typeof core._llamadart_webgpu_decision_capabilities_json !== "function") { | |
| return unsupported("This WebGPU core build does not include decision heads."); | |
| } | |
| const coreVersion = Number( | |
| core.ccall("llamadart_webgpu_decision_api_version", "number", [], []) | |
| ); | |
| if (coreVersion !== DECISION_API_VERSION) { | |
| return unsupported( | |
| `The WebGPU core implements decision API version ${coreVersion}; this bridge needs version ${DECISION_API_VERSION}.` | |
| ); | |
| } | |
| const raw = core.ccall( | |
| "llamadart_webgpu_decision_capabilities_json", | |
| "string", | |
| [], | |
| [] | |
| ) || "{}"; | |
| try { | |
| return JSON.parse(raw); | |
| } catch (_) { | |
| return unsupported("WebGPU decision capability response is invalid"); | |
| } | |
| } | |
| _ensureDecisionDir() { | |
| try { | |
| this._core.FS.mkdir("/decision"); | |
| } catch (_) { | |
| } | |
| } | |
| async _stageDecisionHeadFromUrl(url, headPath, onProgress) { | |
| const fetchTimeoutMs = this._resolveFetchTimeoutMs({}, 18e4); | |
| const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs({}, 9e4); | |
| for (let attempt = 0; ; attempt += 1) { | |
| try { | |
| const response = await this._fetchWithTimeout( | |
| url, | |
| { cache: "no-store" }, | |
| fetchTimeoutMs | |
| ); | |
| if (!response.ok) { | |
| throw new Error( | |
| `Failed to fetch decision head: ${response.status} ${response.statusText}` | |
| ); | |
| } | |
| await writeResponseToFsFileWithProgress( | |
| response, | |
| this._core.FS, | |
| headPath, | |
| typeof onProgress === "function" ? onProgress : null, | |
| { | |
| useBigIntPosition: this._coreVariant === "wasm64", | |
| chunkTimeoutMs | |
| } | |
| ); | |
| return; | |
| } catch (error) { | |
| this._unlinkDecisionFile(headPath); | |
| if (!isRetryableStreamNetworkError(error) || attempt >= 1) { | |
| throw error; | |
| } | |
| this._runtimeNotes.push(`decision_head_fetch_retry:${attempt + 1}`); | |
| } | |
| } | |
| } | |
| async loadDecisionHead(source, options = {}) { | |
| if (!this._core || this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| const configJson = options?.configJson ?? null; | |
| if (configJson !== null && typeof configJson !== "string") { | |
| throw new TypeError("Decision head configJson must be a string."); | |
| } | |
| let bytes = null; | |
| if (typeof source === "string") { | |
| if (source.length === 0) { | |
| throw new Error("Decision head URL is empty."); | |
| } | |
| } else { | |
| bytes = bufferSourceBytes(source); | |
| if (!bytes) { | |
| throw new TypeError("Decision head source must be a URL string, an ArrayBuffer or a typed array."); | |
| } | |
| if (bytes.byteLength === 0) { | |
| throw new Error("Decision head bytes are empty."); | |
| } | |
| } | |
| const capabilities = this.getDecisionCapabilities(); | |
| if (capabilities.supported !== true) { | |
| throw new Error( | |
| capabilities.reason || "The loaded model does not support decision heads." | |
| ); | |
| } | |
| const core = this._core; | |
| this._ensureDecisionDir(); | |
| const taskId = `${Date.now()}_${++this._decisionFileCounter}`; | |
| const headPath = `/decision/head_${taskId}.safetensors`; | |
| const configPath = configJson === null ? null : `/decision/config_${taskId}.json`; | |
| try { | |
| let label = "decision head bytes"; | |
| if (bytes) { | |
| core.FS.writeFile(headPath, bytes); | |
| } else { | |
| label = basenameFromUrl(source).slice(0, 256); | |
| await this._stageDecisionHeadFromUrl(source, headPath, options?.onProgress); | |
| } | |
| if (configPath !== null) { | |
| core.FS.writeFile(configPath, textEncoder.encode(configJson)); | |
| } | |
| const handle = Number( | |
| await core.ccall( | |
| "llamadart_webgpu_decision_load", | |
| "number", | |
| ["string", "string", "string"], | |
| [headPath, label, configPath], | |
| { async: true } | |
| ) | |
| ); | |
| if (handle <= 0) { | |
| throw new Error(this._coreErrorMessage("Failed to load decision head", handle)); | |
| } | |
| const raw = core.ccall( | |
| "llamadart_webgpu_decision_head_info_json", | |
| "string", | |
| ["number"], | |
| [handle] | |
| ) || "{}"; | |
| return JSON.parse(raw); | |
| } finally { | |
| this._unlinkDecisionFile(headPath); | |
| this._unlinkDecisionFile(configPath); | |
| } | |
| } | |
| async runDecision(handle, sequences) { | |
| const nativeHandle = decisionHandleFrom(handle); | |
| const input = encodeDecisionSequences(sequences); | |
| if (!this._core || this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| const core = this._core; | |
| this._ensureDecisionDir(); | |
| const taskId = `${Date.now()}_${++this._decisionFileCounter}`; | |
| const inputPath = `/decision/input_${taskId}.bin`; | |
| const outputPath = `/decision/output_${taskId}.bin`; | |
| try { | |
| core.FS.writeFile(inputPath, input); | |
| const count = Number( | |
| await core.ccall( | |
| "llamadart_webgpu_decision_run", | |
| "number", | |
| ["number", "string", "string"], | |
| [nativeHandle, inputPath, outputPath], | |
| { async: true } | |
| ) | |
| ); | |
| if (count < 0) { | |
| throw new Error(this._coreErrorMessage("Decision run failed", count)); | |
| } | |
| return decodeDecisionOutputs(core.FS.readFile(outputPath), count); | |
| } finally { | |
| this._unlinkDecisionFile(inputPath); | |
| this._unlinkDecisionFile(outputPath); | |
| } | |
| } | |
| async freeDecisionHead(handle) { | |
| const nativeHandle = decisionHandleFrom(handle); | |
| if (!this._core) { | |
| return; | |
| } | |
| await this._core.ccall( | |
| "llamadart_webgpu_decision_free", | |
| "number", | |
| ["number"], | |
| [nativeHandle], | |
| { async: true } | |
| ); | |
| } | |
| getLoraAdapterCapabilities() { | |
| const unsupported = (reason) => ({ | |
| apiVersion: LORA_API_VERSION, | |
| supported: false, | |
| reason | |
| }); | |
| const core = this._core; | |
| if (!core) { | |
| return unsupported("WebGPU core is not initialized"); | |
| } | |
| if (typeof core._llamadart_webgpu_lora_api_version !== "function") { | |
| return unsupported("This WebGPU core build does not include LoRA adapters."); | |
| } | |
| const coreVersion = Number( | |
| core.ccall("llamadart_webgpu_lora_api_version", "number", [], []) | |
| ); | |
| if (coreVersion !== LORA_API_VERSION) { | |
| return unsupported( | |
| `The WebGPU core implements LoRA API version ${coreVersion}; this bridge needs version ${LORA_API_VERSION}.` | |
| ); | |
| } | |
| return { apiVersion: LORA_API_VERSION, supported: true }; | |
| } | |
| _requireLoraCore() { | |
| if (!this._core || this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| const capabilities = this.getLoraAdapterCapabilities(); | |
| if (capabilities.supported !== true) { | |
| throw new Error(capabilities.reason || "This bridge cannot load LoRA adapters."); | |
| } | |
| return this._core; | |
| } | |
| /** | |
| * Downloads a URL adapter into `adapterPath`, through the Cache API unless | |
| * `options.useCache` is false. A cache miss stores the response while it is | |
| * written, so progress starts with the first chunk. | |
| */ | |
| async _stageLoraAdapterFromUrl(url, adapterPath, options) { | |
| const core = this._core; | |
| const signal = options.signal || null; | |
| const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null; | |
| const fetchTimeoutMs = this._resolveFetchTimeoutMs(options, 18e4); | |
| const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs(options, 9e4); | |
| const fetchInit = signal ? { signal } : {}; | |
| const cacheKey = normalizeAbsoluteUrl(url); | |
| let cache = null; | |
| if (options.useCache !== false && typeof globalThis.caches?.open === "function") { | |
| try { | |
| cache = await globalThis.caches.open(this._resolveCacheName(options)); | |
| } catch (_) { | |
| this._runtimeNotes.push("lora_cache_error"); | |
| } | |
| } | |
| for (let attempt = 0; ; attempt += 1) { | |
| throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE); | |
| try { | |
| let response = null; | |
| if (cache) { | |
| try { | |
| response = await cache.match(cacheKey) || null; | |
| } catch (_) { | |
| this._runtimeNotes.push("lora_cache_error"); | |
| } | |
| } | |
| let stored = null; | |
| if (!response) { | |
| response = await this._fetchWithTimeout( | |
| url, | |
| cache ? fetchInit : { cache: "no-store", ...fetchInit }, | |
| fetchTimeoutMs | |
| ); | |
| if (!response.ok) { | |
| throw new Error( | |
| `Failed to fetch LoRA adapter: ${response.status} ${response.statusText}` | |
| ); | |
| } | |
| if (cache) { | |
| stored = cache.put(cacheKey, response.clone()).then( | |
| () => { | |
| this._runtimeNotes.push("lora_cache_stored"); | |
| }, | |
| () => { | |
| this._runtimeNotes.push("lora_cache_store_failed"); | |
| } | |
| ); | |
| } | |
| } | |
| await writeResponseToFsFileWithProgress( | |
| response, | |
| core.FS, | |
| adapterPath, | |
| progressCallback, | |
| { | |
| useBigIntPosition: this._coreVariant === "wasm64", | |
| chunkTimeoutMs, | |
| signal, | |
| abortMessage: LORA_LOAD_ABORT_MESSAGE | |
| } | |
| ); | |
| await stored; | |
| return; | |
| } catch (error) { | |
| this._deleteFsFile(adapterPath); | |
| throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE); | |
| if (!isRetryableStreamNetworkError(error) || attempt >= 1) { | |
| throw error; | |
| } | |
| this._runtimeNotes.push(`lora_fetch_retry:${attempt + 1}`); | |
| } | |
| } | |
| } | |
| async loadLoraAdapter(source, options = {}) { | |
| let bytes = null; | |
| if (typeof source === "string") { | |
| if (source.length === 0) { | |
| throw new Error("LoRA adapter URL is empty."); | |
| } | |
| } else { | |
| bytes = bufferSourceBytes(source); | |
| if (!bytes) { | |
| throw new TypeError("LoRA adapter source must be a URL string, an ArrayBuffer or a typed array."); | |
| } | |
| if (bytes.byteLength === 0) { | |
| throw new Error("LoRA adapter bytes are empty."); | |
| } | |
| } | |
| const core = this._requireLoraCore(); | |
| const signal = options.signal || null; | |
| throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE); | |
| ensureFsDirectory(core.FS, "/lora"); | |
| const adapterPath = `/lora/adapter_${++this._loraFileCounter}.gguf`; | |
| try { | |
| if (bytes) { | |
| core.FS.writeFile(adapterPath, bytes); | |
| } else { | |
| await this._stageLoraAdapterFromUrl(source, adapterPath, options); | |
| } | |
| throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE); | |
| const handle = Number( | |
| await core.ccall( | |
| "llamadart_webgpu_lora_load", | |
| "number", | |
| ["string"], | |
| [adapterPath], | |
| { async: true } | |
| ) | |
| ); | |
| if (handle <= 0) { | |
| throw new Error(this._coreErrorMessage("Failed to load LoRA adapter", handle)); | |
| } | |
| this._loraAdaptersLoaded = true; | |
| return { handle }; | |
| } finally { | |
| this._deleteFsFile(adapterPath); | |
| } | |
| } | |
| async _callLoraCore(name, argTypes, args, failure) { | |
| const core = this._requireLoraCore(); | |
| const rc = Number(await core.ccall(name, "number", argTypes, args, { async: true })); | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage(failure, rc)); | |
| } | |
| } | |
| async setLoraAdapter(handle, scale = 1) { | |
| await this._callLoraCore( | |
| "llamadart_webgpu_lora_set", | |
| ["number", "number"], | |
| [loraHandleFrom(handle), loraScaleFrom(scale)], | |
| "Failed to apply LoRA adapter" | |
| ); | |
| } | |
| async removeLoraAdapter(handle) { | |
| await this._callLoraCore( | |
| "llamadart_webgpu_lora_remove", | |
| ["number"], | |
| [loraHandleFrom(handle)], | |
| "Failed to remove LoRA adapter" | |
| ); | |
| } | |
| async clearLoraAdapters() { | |
| await this._callLoraCore( | |
| "llamadart_webgpu_lora_clear", | |
| [], | |
| [], | |
| "Failed to clear LoRA adapters" | |
| ); | |
| } | |
| _clearStagedMediaFiles() { | |
| if (!this._core || this._stagedMediaPaths.length === 0) { | |
| this._stagedMediaPaths = []; | |
| return; | |
| } | |
| for (const mediaPath of this._stagedMediaPaths) { | |
| try { | |
| this._core.FS.unlink(mediaPath); | |
| } catch (_) { | |
| } | |
| } | |
| this._stagedMediaPaths = []; | |
| } | |
| _clearPendingMedia() { | |
| this._core?.ccall("llamadart_webgpu_media_clear_pending", null, [], []); | |
| this._clearStagedMediaFiles(); | |
| } | |
| _persistMediaBytes(bytes, extension = ".bin") { | |
| if (!this._core) { | |
| throw new Error("WebGPU core is not initialized."); | |
| } | |
| try { | |
| this._core.FS.mkdir("/media"); | |
| } catch (_) { | |
| } | |
| this._mediaFileCounter += 1; | |
| const suffix = typeof extension === "string" && extension.startsWith(".") ? extension : ".bin"; | |
| const mediaPath = `/media/input_${Date.now()}_${this._mediaFileCounter}${suffix}`; | |
| this._core.FS.writeFile(mediaPath, bytes); | |
| this._stagedMediaPaths.push(mediaPath); | |
| return mediaPath; | |
| } | |
| _addMediaFile(mediaPath) { | |
| const rc = Number( | |
| this._core.ccall( | |
| "llamadart_webgpu_media_add_file", | |
| "number", | |
| ["string"], | |
| [mediaPath] | |
| ) | |
| ); | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to add media file", rc)); | |
| } | |
| } | |
| _addRawRgbMediaBytes(bytes, width, height) { | |
| const useHeapBuffer = this._core && typeof this._core._malloc === "function" && typeof this._core._free === "function" && this._core.HEAPU8 && typeof this._core.HEAPU8.set === "function"; | |
| let rc = 0; | |
| if (useHeapBuffer) { | |
| const ptr = this._core._malloc(bytes.length); | |
| if (!Number.isFinite(ptr) || ptr <= 0) { | |
| throw new Error("Failed to allocate core heap buffer for RGB media bytes"); | |
| } | |
| try { | |
| this._core.HEAPU8.set(bytes, ptr); | |
| rc = Number( | |
| this._core.ccall( | |
| "llamadart_webgpu_media_add_rgb", | |
| "number", | |
| ["number", "number", "number", "number"], | |
| [width, height, ptr, bytes.length] | |
| ) | |
| ); | |
| } finally { | |
| this._core._free(ptr); | |
| } | |
| } else { | |
| rc = Number( | |
| this._core.ccall( | |
| "llamadart_webgpu_media_add_rgb", | |
| "number", | |
| ["number", "number", "array", "number"], | |
| [width, height, bytes, bytes.length] | |
| ) | |
| ); | |
| } | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to add raw RGB media bytes", rc)); | |
| } | |
| } | |
| _addAudioSamples(samples) { | |
| const sampleBytes = new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength); | |
| const useHeapBuffer = this._core && typeof this._core._malloc === "function" && typeof this._core._free === "function" && this._core.HEAPU8 && typeof this._core.HEAPU8.set === "function"; | |
| let rc = 0; | |
| if (useHeapBuffer) { | |
| const ptr = this._core._malloc(sampleBytes.length); | |
| if (!Number.isFinite(ptr) || ptr <= 0) { | |
| throw new Error("Failed to allocate core heap buffer for audio samples"); | |
| } | |
| try { | |
| this._core.HEAPU8.set(sampleBytes, ptr); | |
| rc = Number( | |
| this._core.ccall( | |
| "llamadart_webgpu_media_add_audio_f32", | |
| "number", | |
| ["number", "number"], | |
| [ptr, samples.length] | |
| ) | |
| ); | |
| } finally { | |
| this._core._free(ptr); | |
| } | |
| } else { | |
| rc = Number( | |
| this._core.ccall( | |
| "llamadart_webgpu_media_add_audio_f32", | |
| "number", | |
| ["array", "number"], | |
| [sampleBytes, samples.length] | |
| ) | |
| ); | |
| } | |
| if (rc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to add audio samples", rc)); | |
| } | |
| } | |
| async _fetchMediaBytes(url) { | |
| const response = await this._fetchWithTimeout( | |
| url, | |
| { cache: "no-store" }, | |
| this._resolveFetchTimeoutMs({}, 12e4) | |
| ); | |
| if (!response.ok) { | |
| throw new Error(`Failed to fetch media: ${response.status} ${response.statusText}`); | |
| } | |
| return new Uint8Array(await response.arrayBuffer()); | |
| } | |
| async _prepareImageBytesForMultimodal(bytes, options = {}) { | |
| const maxPixels = this._resolveMediaImageMaxPixels(options); | |
| const maxEdge = this._resolveMediaImageMaxEdge(options); | |
| if (maxPixels <= 0 && maxEdge <= 0) { | |
| return null; | |
| } | |
| return decodeImageBytesToRgb(bytes, { | |
| maxPixels, | |
| maxEdge | |
| }); | |
| } | |
| async _stageMultimodalParts(parts, options = {}) { | |
| this._clearPendingMedia(); | |
| const mediaParts = Array.isArray(parts) ? parts : []; | |
| if (mediaParts.length === 0) { | |
| return; | |
| } | |
| if (!this._mmProjPath) { | |
| throw new Error( | |
| "Multimodal input requires a loaded projector. Call loadMultimodalProjector first." | |
| ); | |
| } | |
| for (const rawPart of mediaParts) { | |
| const part = rawPart && typeof rawPart === "object" ? rawPart : {}; | |
| const type = String(part.type || "").toLowerCase(); | |
| if (type === "image") { | |
| const bytes = toUint8Array(part.bytes); | |
| if (bytes && bytes.length > 0) { | |
| const width = Number(part.width); | |
| const height = Number(part.height); | |
| const isRawRgb = Number.isInteger(width) && Number.isInteger(height) && width > 0 && height > 0 && bytes.length === width * height * 3; | |
| if (isRawRgb) { | |
| this._addRawRgbMediaBytes(bytes, width, height); | |
| } else { | |
| const prepared2 = await this._prepareImageBytesForMultimodal(bytes, options); | |
| if (prepared2 && prepared2.bytes && prepared2.bytes.length > 0) { | |
| const mediaPath = this._persistMediaBytes(prepared2.bytes, ".img"); | |
| this._addMediaFile(mediaPath); | |
| if (prepared2.resized) { | |
| this._runtimeNotes.push( | |
| `media_image_resized:${prepared2.sourceWidth}x${prepared2.sourceHeight}->${prepared2.width}x${prepared2.height}` | |
| ); | |
| } | |
| } else { | |
| const mediaPath = this._persistMediaBytes(bytes, ".img"); | |
| this._addMediaFile(mediaPath); | |
| } | |
| } | |
| continue; | |
| } | |
| if (typeof part.url !== "string" || part.url.length === 0) { | |
| throw new Error("Image part must provide bytes or url."); | |
| } | |
| const fetched = await this._fetchMediaBytes(part.url); | |
| const prepared = await this._prepareImageBytesForMultimodal(fetched, options); | |
| if (prepared && prepared.bytes && prepared.bytes.length > 0) { | |
| const mediaPath = this._persistMediaBytes(prepared.bytes, ".img"); | |
| this._addMediaFile(mediaPath); | |
| if (prepared.resized) { | |
| this._runtimeNotes.push( | |
| `media_image_resized:${prepared.sourceWidth}x${prepared.sourceHeight}->${prepared.width}x${prepared.height}` | |
| ); | |
| } | |
| } else { | |
| const mediaPath = this._persistMediaBytes(fetched, ".img"); | |
| this._addMediaFile(mediaPath); | |
| } | |
| continue; | |
| } | |
| if (type === "audio") { | |
| const samples = toFloat32Array(part.samples); | |
| if (samples && samples.length > 0) { | |
| this._addAudioSamples(samples); | |
| continue; | |
| } | |
| const bytes = toUint8Array(part.bytes); | |
| if (bytes && bytes.length > 0) { | |
| const mediaPath2 = this._persistMediaBytes(bytes, ".aud"); | |
| this._addMediaFile(mediaPath2); | |
| continue; | |
| } | |
| if (typeof part.url !== "string" || part.url.length === 0) { | |
| throw new Error("Audio part must provide samples, bytes, or url."); | |
| } | |
| const fetched = await this._fetchMediaBytes(part.url); | |
| const mediaPath = this._persistMediaBytes(fetched, ".aud"); | |
| this._addMediaFile(mediaPath); | |
| } | |
| } | |
| } | |
| getCompletionCapabilities() { | |
| const core = this._core; | |
| if (!core || typeof core._llamadart_webgpu_completion_capabilities_json !== "function") { | |
| return { ...NO_COMPLETION_CAPABILITIES }; | |
| } | |
| return completionCapabilitiesFrom( | |
| core.ccall("llamadart_webgpu_completion_capabilities_json", "string", [], []) | |
| ); | |
| } | |
| async createCompletion(prompt, options = {}) { | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| const sampling = resolveCompletionSamplingOptions(options); | |
| requireCompletionCapabilities(sampling, this.getCompletionCapabilities()); | |
| this._abortRequested = false; | |
| const startedAt = performance.now(); | |
| let firstTokenAt = null; | |
| let nPredict = Number(options.nPredict) > 0 ? Number(options.nPredict) : 256; | |
| const hasMediaParts = Array.isArray(options.parts) && options.parts.length > 0; | |
| if (hasMediaParts) { | |
| const requestedMediaCap = Number(options.mediaMaxPredict); | |
| const mediaCap = Number.isFinite(requestedMediaCap) && requestedMediaCap > 0 ? Math.max(32, Math.trunc(requestedMediaCap)) : 256; | |
| if (nPredict > mediaCap) { | |
| nPredict = mediaCap; | |
| this._runtimeNotes.push(`media_n_predict_capped:${mediaCap}`); | |
| } | |
| } | |
| const temp = Number.isFinite(options.temp) ? Number(options.temp) : 0.8; | |
| const topK = Number.isFinite(options.topK) ? Number(options.topK) : 40; | |
| const topP = Number.isFinite(options.topP) ? Number(options.topP) : 0.95; | |
| const penalty = Number.isFinite(options.penalty) ? Number(options.penalty) : 1.1; | |
| const grammar = typeof options.grammar === "string" && options.grammar.length > 0 ? options.grammar : null; | |
| const seed = Number.isInteger(options.seed) ? Number(options.seed) : Math.floor(Math.random() * 4294967295); | |
| await this._stageMultimodalParts(options.parts, options); | |
| let generationStarted = false; | |
| const speculativePaths = []; | |
| try { | |
| if (sampling.speculative) { | |
| await this._setNextSpeculative(sampling.speculative, nPredict, speculativePaths); | |
| } | |
| const beginRc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_begin_generation", | |
| "number", | |
| [ | |
| "string", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "string", | |
| "number", | |
| "number", | |
| "number", | |
| "number", | |
| "string", | |
| "string", | |
| "string" | |
| ], | |
| [ | |
| String(prompt), | |
| temp, | |
| topK, | |
| topP, | |
| penalty, | |
| grammar, | |
| seed >>> 0, | |
| sampling.minP, | |
| sampling.presencePenalty, | |
| sampling.thinkingBudget?.maxTokens ?? 0, | |
| sampling.thinkingBudget?.startTag ?? null, | |
| sampling.thinkingBudget?.endTag ?? null, | |
| sampling.thinkingBudget?.forcedMessage ?? null | |
| ], | |
| { async: true } | |
| ) | |
| ); | |
| if (beginRc === GENERATION_ALREADY_ACTIVE_RC) { | |
| throw new Error(GENERATION_ALREADY_ACTIVE_MESSAGE); | |
| } | |
| if (beginRc !== 0) { | |
| throw new Error(this._coreErrorMessage("Failed to start generation", beginRc)); | |
| } | |
| generationStarted = true; | |
| for (const path of speculativePaths.splice(0)) { | |
| this._deleteFsFile(path); | |
| } | |
| let generated = 0; | |
| const shouldEmitCurrentText = options.emitCurrentTextOnToken !== false; | |
| const tokenEventEncoding = typeof options.tokenEventEncoding === "string" ? String(options.tokenEventEncoding || "").toLowerCase() : "bytes"; | |
| const emitTokenText = tokenEventEncoding === "text"; | |
| const yieldInterval = 4; | |
| let streamed = ""; | |
| let emittedStableText = ""; | |
| let finishReason = "length"; | |
| while (generated < nPredict) { | |
| if (this._abortRequested || options.signal?.aborted) { | |
| finishReason = "cancelled"; | |
| break; | |
| } | |
| const stepRc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_next_token", | |
| "number", | |
| [], | |
| [], | |
| { async: true } | |
| ) | |
| ); | |
| if (stepRc === 0) { | |
| finishReason = this._abortRequested || options.signal?.aborted ? "cancelled" : "stop"; | |
| break; | |
| } | |
| if (stepRc < 0) { | |
| const stepErrorText = this._coreErrorMessage("Generation step failed", stepRc); | |
| if (generated > 0 && this._isContextLimitGenerationError(stepErrorText)) { | |
| this._runtimeNotes.push("generation_stopped_context_limit"); | |
| this._emitLogger( | |
| "warn", | |
| "warning: generation reached context/memory limit; returning partial output." | |
| ); | |
| break; | |
| } | |
| if (sampling.speculative == null && this._shouldAttemptGenerationRecovery(stepErrorText, options, generated)) { | |
| const recovered = await this._recoverGenerationWithCpuFallback(options); | |
| if (recovered) { | |
| if (generationStarted) { | |
| try { | |
| this._core.ccall("llamadart_webgpu_end_generation", null, [], []); | |
| } catch (_) { | |
| } | |
| generationStarted = false; | |
| } | |
| this._clearPendingMedia(); | |
| const retryOptions = { | |
| ...options, | |
| _llamadartGenerationRecoveryAttempted: true | |
| }; | |
| return await this.createCompletion(prompt, retryOptions); | |
| } | |
| } | |
| throw new Error(stepErrorText); | |
| } | |
| generated += 1; | |
| const fullText = this._core.ccall("llamadart_webgpu_last_output", "string", [], []) || ""; | |
| streamed = fullText; | |
| const stableText = trimUnstableUtf8Tail(fullText); | |
| if (!stableText.startsWith(emittedStableText)) { | |
| emittedStableText = ""; | |
| } | |
| const deltaText = stableText.slice(emittedStableText.length); | |
| if (deltaText.length === 0) { | |
| continue; | |
| } | |
| emittedStableText = stableText; | |
| firstTokenAt ??= performance.now(); | |
| if (typeof options.onToken === "function") { | |
| const piecePayload = emitTokenText ? deltaText : textEncoder.encode(deltaText); | |
| options.onToken(piecePayload, shouldEmitCurrentText ? fullText : null); | |
| } | |
| if (yieldInterval > 0 && generated % yieldInterval === 0) { | |
| await new Promise((resolve) => setTimeout(resolve, 0)); | |
| } | |
| } | |
| const text = this._core.ccall("llamadart_webgpu_last_output", "string", [], []) || streamed || ""; | |
| const tailText = text.startsWith(emittedStableText) ? text.slice(emittedStableText.length) : ""; | |
| if (tailText.length > 0) { | |
| firstTokenAt ??= performance.now(); | |
| if (typeof options.onToken === "function") { | |
| const piecePayload = emitTokenText ? tailText : textEncoder.encode(tailText); | |
| options.onToken(piecePayload, shouldEmitCurrentText ? text : null); | |
| } | |
| } | |
| if (typeof options.onUsage === "function") { | |
| const counts = JSON.parse( | |
| this._core.ccall("llamadart_webgpu_last_generation_usage_json", "string", [], []) || "{}" | |
| ); | |
| const speculative = counts?.speculative; | |
| options.onUsage({ | |
| promptTokens: Number(counts?.promptTokens) || 0, | |
| cachedPromptTokens: Number(counts?.cachedPromptTokens) || 0, | |
| completionTokens: Number(counts?.completionTokens) || 0, | |
| timeToFirstTokenMs: firstTokenAt == null ? null : firstTokenAt - startedAt, | |
| durationMs: performance.now() - startedAt, | |
| finishReason, | |
| ...speculative && typeof speculative === "object" ? { | |
| speculative: { | |
| draftTokens: Number(speculative.draftTokens) || 0, | |
| acceptedDraftTokens: Number(speculative.acceptedDraftTokens) || 0, | |
| draftAttempts: Number(speculative.draftAttempts) || 0, | |
| verifyTokens: Number(speculative.verifyTokens) || 0, | |
| replayTokens: Number(speculative.replayTokens) || 0 | |
| } | |
| } : {} | |
| }); | |
| } | |
| return text; | |
| } finally { | |
| if (generationStarted) { | |
| await this._core.ccall("llamadart_webgpu_end_generation", null, [], [], { async: true }); | |
| } | |
| for (const path of speculativePaths) { | |
| this._deleteFsFile(path); | |
| } | |
| this._clearPendingMedia(); | |
| } | |
| } | |
| async tokenize(text, _addSpecial = true) { | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| const rc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_tokenize_to_json", | |
| "number", | |
| ["string", "number"], | |
| [String(text), _addSpecial ? 1 : 0], | |
| { async: true } | |
| ) | |
| ); | |
| if (rc < 0) { | |
| throw new Error(this._coreErrorMessage("Tokenization failed", rc)); | |
| } | |
| const raw = this._core.ccall("llamadart_webgpu_last_tokens_json", "string", [], []) || "[]"; | |
| const parsed = JSON.parse(raw); | |
| return Array.isArray(parsed) ? parsed.map((v) => Number(v) | 0) : []; | |
| } | |
| _ensureStateDir() { | |
| const core = this._core; | |
| if (!core?.FS) { | |
| throw new Error("Bridge filesystem is not initialized"); | |
| } | |
| ensureFsDirectory(core.FS, "/states"); | |
| } | |
| _normalizeStateTokens(tokens) { | |
| const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); | |
| return normalized.map((value) => Number(value) | 0); | |
| } | |
| _nextStateTempPath() { | |
| this._ensureStateDir(); | |
| this._stateFileCounter += 1; | |
| return `/states/state_${Date.now()}_${this._stateFileCounter}.bin`; | |
| } | |
| async stateSaveFile(path, tokens = []) { | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| if (typeof path !== "string" || path.length === 0) { | |
| throw new Error("State file path is empty."); | |
| } | |
| const normalized = this._normalizeStateTokens(tokens); | |
| const tokenText = JSON.stringify(normalized); | |
| const rc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_state_save_file", | |
| "number", | |
| ["string", "string"], | |
| [path, tokenText], | |
| { async: true } | |
| ) | |
| ); | |
| if (rc < 0) { | |
| throw new Error(this._coreErrorMessage("State save failed", rc)); | |
| } | |
| return true; | |
| } | |
| async stateLoadFile(path, tokenCapacity = this.getContextSize()) { | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| if (typeof path !== "string" || path.length === 0) { | |
| throw new Error("State file path is empty."); | |
| } | |
| const capacity = Number(tokenCapacity) > 0 ? Math.trunc(Number(tokenCapacity)) : this.getContextSize(); | |
| const rc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_state_load_file", | |
| "number", | |
| ["string", "number"], | |
| [path, capacity], | |
| { async: true } | |
| ) | |
| ); | |
| if (rc < 0) { | |
| throw new Error(this._coreErrorMessage("State load failed", rc)); | |
| } | |
| const raw = this._core.ccall("llamadart_webgpu_last_tokens_json", "string", [], []) || "[]"; | |
| const parsed = JSON.parse(raw); | |
| const restoredTokens = Array.isArray(parsed) ? parsed.map((v) => Number(v) | 0) : []; | |
| return { tokens: restoredTokens }; | |
| } | |
| async stateSaveBytes(tokens = []) { | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| const tempPath = this._nextStateTempPath(); | |
| try { | |
| await this.stateSaveFile(tempPath, tokens); | |
| const bytes = this._core.FS.readFile(tempPath); | |
| return bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes || []); | |
| } finally { | |
| this._deleteFsFile(tempPath); | |
| } | |
| } | |
| async stateLoadBytes(bytes, tokenCapacity = this.getContextSize()) { | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| const normalizedBytes = toUint8Array(bytes); | |
| if (!normalizedBytes || normalizedBytes.length === 0) { | |
| throw new Error("State bytes are empty."); | |
| } | |
| const tempPath = this._nextStateTempPath(); | |
| try { | |
| this._core.FS.writeFile(tempPath, normalizedBytes); | |
| return await this.stateLoadFile(tempPath, tokenCapacity); | |
| } finally { | |
| this._deleteFsFile(tempPath); | |
| } | |
| } | |
| async detokenize(tokens, _special = false) { | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); | |
| const tokenText = JSON.stringify(normalized.map((v) => Number(v) | 0)); | |
| const rc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_detokenize_from_json", | |
| "number", | |
| ["string", "number"], | |
| [tokenText, _special ? 1 : 0], | |
| { async: true } | |
| ) | |
| ); | |
| if (rc < 0) { | |
| throw new Error(this._coreErrorMessage("Detokenization failed", rc)); | |
| } | |
| return this._core.ccall("llamadart_webgpu_last_detokenized", "string", [], []) || ""; | |
| } | |
| _embeddingAssetVersion() { | |
| return embeddingAssetVersion(this._coreVariant === "wasm64" ? this._config.coreModuleUrlMem64 : this._config.coreModuleUrl); | |
| } | |
| async getEmbeddingCapabilities() { | |
| if (this._modelBytes <= 0) return { ...EMPTY_EMBEDDING_CAPABILITIES }; | |
| if (typeof this._core?._llamadart_webgpu_embedding_capabilities_json !== "function" || typeof this._core?._llamadart_webgpu_embed_parts_to_json !== "function") { | |
| throw missingEmbeddingCapability(this._embeddingAssetVersion()); | |
| } | |
| return embeddingCapabilitiesFrom( | |
| this._core.ccall("llamadart_webgpu_embedding_capabilities_json", "string", [], []), | |
| this._embeddingAssetVersion() | |
| ); | |
| } | |
| async _preflightEmbeddingInputs(inputs) { | |
| if (!inputs.some(Array.isArray)) return; | |
| const caps = await this.getEmbeddingCapabilities(); | |
| for (const input of inputs) requireEmbeddingModalities(input, caps, !!this._mmProjPath); | |
| } | |
| async _embedParts(parts, options) { | |
| const prompt = parts.map((part) => part.type === "text" ? part.text : EMBEDDING_MEDIA_MARKER).join(""); | |
| const promptBytes = textEncoder.encode(prompt); | |
| const media = parts.filter((part) => part.type !== "text"); | |
| try { | |
| await this._stageMultimodalParts(media); | |
| if (this._abortRequested) throw createAbortError("Embedding was cancelled."); | |
| const rc = Number(await this._core.ccall( | |
| "llamadart_webgpu_embed_parts_to_json", | |
| "number", | |
| ["array", "number", "number"], | |
| [promptBytes, options?.normalize !== false ? 1 : 0, promptBytes.length], | |
| { async: true } | |
| )); | |
| if (this._abortRequested) throw createAbortError("Embedding was cancelled."); | |
| if (rc < 0) throw embeddingCoreError(rc, this._embeddingAssetVersion()); | |
| const raw = this._core.ccall("llamadart_webgpu_last_embedding_json", "string", [], []) || "[]"; | |
| const parsed = JSON.parse(raw); | |
| return Array.isArray(parsed) ? parsed.map((v) => Number.isFinite(Number(v)) ? Number(v) : 0) : []; | |
| } catch (error) { | |
| if (error instanceof WebAssembly.RuntimeError) throw error; | |
| if (error instanceof EmbeddingError || error?.name === "AbortError") throw error; | |
| throw new EmbeddingError("EMBEDDING_FAILED", "Embedding media preparation or execution failed."); | |
| } finally { | |
| this._clearPendingMedia(); | |
| } | |
| } | |
| async embed(text, options = {}) { | |
| if (Array.isArray(text)) { | |
| const parts = snapshotEmbedInput(text); | |
| this._abortRequested = false; | |
| await this._preflightEmbeddingInputs([parts]); | |
| if (this._abortRequested) throw createAbortError("Embedding was cancelled."); | |
| return this._embedParts(parts, options); | |
| } | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| const normalize = options?.normalize !== false; | |
| const rc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_embed_to_json", | |
| "number", | |
| ["string", "number"], | |
| [String(text), normalize ? 1 : 0], | |
| { async: true } | |
| ) | |
| ); | |
| if (rc < 0) { | |
| throw new Error(this._coreErrorMessage("Embedding generation failed", rc)); | |
| } | |
| const raw = this._core.ccall("llamadart_webgpu_last_embedding_json", "string", [], []) || "[]"; | |
| const parsed = JSON.parse(raw); | |
| return Array.isArray(parsed) ? parsed.map((v) => { | |
| const numeric = Number(v); | |
| return Number.isFinite(numeric) ? numeric : 0; | |
| }) : []; | |
| } | |
| async embedBatch(texts, options = {}) { | |
| const normalized = Array.isArray(texts) ? texts : Array.from(texts || []); | |
| if (normalized.length === 0) { | |
| return []; | |
| } | |
| const inputs = normalized.map((input) => typeof input === "string" ? input : snapshotEmbedInput(input)); | |
| this._abortRequested = false; | |
| await this._preflightEmbeddingInputs(inputs); | |
| const normalize = options?.normalize !== false; | |
| const vectors = []; | |
| for (const text of inputs) { | |
| if (this._abortRequested) throw createAbortError("Batch embedding was cancelled."); | |
| vectors.push(await (typeof text === "string" ? this.embed(text, { normalize }) : this._embedParts(text, { normalize }))); | |
| } | |
| return vectors; | |
| } | |
| async scoreNextToken(prompt, options = {}) { | |
| if (this._modelBytes <= 0) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| if (typeof prompt !== "string") { | |
| throw new TypeError("Next-token scoring prompt must be a string."); | |
| } | |
| const candidates = nextTokenCandidateIds(options?.candidates); | |
| const topK = options?.topK ?? 0; | |
| if (!isInt32(topK)) { | |
| throw new TypeError(`Next-token topK is ${String(topK)}; expected a 32-bit integer.`); | |
| } | |
| const reusePromptPrefix = options?.reusePromptPrefix !== false; | |
| const rc = Number( | |
| await this._core.ccall( | |
| "llamadart_webgpu_score_next_token_to_json", | |
| "number", | |
| ["string", "string", "number", "number"], | |
| [prompt, JSON.stringify(candidates), topK, reusePromptPrefix ? 1 : 0], | |
| { async: true } | |
| ) | |
| ); | |
| if (rc < 0) { | |
| throw new Error(this._coreErrorMessage("Next-token scoring failed", rc)); | |
| } | |
| const raw = this._core.ccall("llamadart_webgpu_last_next_token_scores_json", "string", [], []) || "{}"; | |
| const parsed = JSON.parse(raw); | |
| return { | |
| candidates: scoredTokensFrom(parsed?.candidates), | |
| top: scoredTokensFrom(parsed?.top), | |
| promptTokens: Number(parsed?.promptTokens) || 0 | |
| }; | |
| } | |
| getModelMetadata() { | |
| let modelMetadata = {}; | |
| try { | |
| const raw = this._core?.ccall("llamadart_webgpu_model_meta_json", "string", [], []); | |
| if (raw) { | |
| const parsed = JSON.parse(raw); | |
| if (parsed && typeof parsed === "object") { | |
| modelMetadata = parsed; | |
| } | |
| } | |
| } catch (_) { | |
| } | |
| return { | |
| ...modelMetadata, | |
| "llamadart.webgpu.prototype": "1", | |
| "llamadart.webgpu.backends": this._backendLabels.join(","), | |
| "llamadart.webgpu.model_bytes": String(this._modelBytes), | |
| "llamadart.webgpu.n_threads": String(this._threads), | |
| "llamadart.webgpu.n_threads_batch": String(this._threadsBatch), | |
| "llamadart.webgpu.n_batch": this._nBatch > 0 ? String(this._nBatch) : "", | |
| "llamadart.webgpu.n_ubatch": this._nUbatch > 0 ? String(this._nUbatch) : "", | |
| "llamadart.webgpu.n_seq_max": this._nSeqMax > 0 ? String(this._nSeqMax) : "", | |
| "llamadart.webgpu.flash_attention": String(this._flashAttention), | |
| "llamadart.webgpu.cache_type_k": String(this._cacheTypeK), | |
| "llamadart.webgpu.cache_type_v": String(this._cacheTypeV), | |
| "llamadart.webgpu.kv_unified": this._kvUnified >= 0 ? String(this._kvUnified) : "", | |
| "llamadart.webgpu.rope_freq_base": this._ropeFrequencyBase > 0 ? String(this._ropeFrequencyBase) : "", | |
| "llamadart.webgpu.rope_freq_scale": this._ropeFrequencyScale > 0 ? String(this._ropeFrequencyScale) : "", | |
| "llamadart.webgpu.split_mode": this._splitMode >= 0 ? String(this._splitMode) : "", | |
| "llamadart.webgpu.main_gpu": this._mainGpu >= 0 ? String(this._mainGpu) : "", | |
| "llamadart.webgpu.thread_pool_size": Number.isFinite(this._threadPoolSizeHint) && this._threadPoolSizeHint > 0 ? String(this._threadPoolSizeHint) : "", | |
| "llamadart.webgpu.n_gpu_layers": String(this._nGpuLayers), | |
| "llamadart.webgpu.core_variant": this._coreVariant, | |
| "llamadart.webgpu.model_source": this._modelSource, | |
| "llamadart.webgpu.model_cache_state": this._modelCacheState, | |
| "llamadart.webgpu.model_cache_name": this._modelCacheName, | |
| "llamadart.webgpu.runtime_notes": this._runtimeNotes.join(";"), | |
| "llamadart.webgpu.mmproj_loaded": this._mmProjPath ? "1" : "0", | |
| "llamadart.webgpu.supports_vision": this._mmSupportsVision ? "1" : "0", | |
| "llamadart.webgpu.supports_audio": this._mmSupportsAudio ? "1" : "0" | |
| }; | |
| } | |
| getContextSize() { | |
| try { | |
| const nctx = Number(this._core?.ccall("llamadart_webgpu_get_context_size", "number", [], [])); | |
| if (nctx > 0) { | |
| return nctx; | |
| } | |
| } catch (_) { | |
| } | |
| return this._nCtx; | |
| } | |
| isGpuActive() { | |
| return this._gpuActive; | |
| } | |
| getBackendName() { | |
| if (this._nGpuLayers === 0) { | |
| return "WASM (Prototype bridge)"; | |
| } | |
| if (this._backendLabels.length > 0) { | |
| return this._backendLabels.join(", "); | |
| } | |
| return this._gpuActive ? "WebGPU (Prototype bridge)" : "WASM (Prototype bridge)"; | |
| } | |
| setLogLevel(level) { | |
| if (Number.isFinite(level)) { | |
| this._logLevel = Math.max(0, Math.min(4, Math.trunc(level))); | |
| } | |
| this._applyCoreLogLevel(); | |
| } | |
| cancel() { | |
| this._abortRequested = true; | |
| if (this._activeTransferAbortController) { | |
| try { | |
| this._activeTransferAbortController.abort(); | |
| } catch (_) { | |
| } | |
| this._activeTransferAbortController = null; | |
| } | |
| try { | |
| this._core?.ccall("llamadart_webgpu_request_cancel", null, [], []); | |
| } catch (_) { | |
| } | |
| } | |
| async dispose() { | |
| if (this._textToSpeechActive && this._textToSpeechDone) { | |
| this.cancel(); | |
| await this._textToSpeechDone; | |
| } | |
| if (this._activeTransferAbortController) { | |
| try { | |
| this._activeTransferAbortController.abort(); | |
| } catch (_) { | |
| } | |
| this._activeTransferAbortController = null; | |
| } | |
| if (this._core) { | |
| this._clearPendingMedia(); | |
| const mmprojPath = this._mmProjPath; | |
| const mmprojFreeRc = Number( | |
| this._core.ccall("llamadart_webgpu_mmproj_free", "number", [], []) | |
| ); | |
| this._core.ccall("llamadart_webgpu_shutdown", null, [], []); | |
| this._deleteFsFile(mmprojPath); | |
| if (mmprojFreeRc !== 0) { | |
| this._emitLogger( | |
| "warn", | |
| this._coreErrorMessage("Failed to unload multimodal projector during dispose", mmprojFreeRc) | |
| ); | |
| } | |
| } | |
| this._modelPath = null; | |
| this._modelPaths = []; | |
| this._modelBytes = 0; | |
| this._modelSource = "network"; | |
| this._modelCacheState = "disabled"; | |
| this._loadedModelUrl = null; | |
| this._mmProjPath = null; | |
| this._mmProjSourceUrl = null; | |
| this._mmSupportsVision = false; | |
| this._mmSupportsAudio = false; | |
| this._loraAdaptersLoaded = false; | |
| this._abortRequested = false; | |
| this._textToSpeechActive = false; | |
| this._textToSpeechDone = null; | |
| this._resolveTextToSpeechDone = null; | |
| this._activeTransferAbortController = null; | |
| this._suppressedWarmupWarningCount = 0; | |
| this._didReportWarmupWarningSuppression = false; | |
| } | |
| async applyChatTemplate(messages, addAssistant = true, _customTemplate = null) { | |
| return buildPromptFromMessages(messages, addAssistant); | |
| } | |
| }; | |
| // js/src/worker_protocol.ts | |
| function serializeWorkerError(error) { | |
| if (!error) { | |
| return "Unknown worker error"; | |
| } | |
| if (typeof error === "string") { | |
| return error; | |
| } | |
| if (typeof error.message === "string" && error.message.length > 0) { | |
| return error.message; | |
| } | |
| try { | |
| return JSON.stringify(error); | |
| } catch (_) { | |
| return String(error); | |
| } | |
| } | |
| var bridgeWorkerModeParam = "__llamadartBridgeWorker"; | |
| function emptyBridgeState() { | |
| return { | |
| metadata: {}, | |
| contextSize: 0, | |
| gpuActive: false, | |
| backendName: "WASM (Prototype bridge)", | |
| supportsVision: false, | |
| supportsAudio: false | |
| }; | |
| } | |
| function snapshotBridgeState(target) { | |
| return { | |
| metadata: target.getModelMetadata(), | |
| contextSize: target.getContextSize(), | |
| gpuActive: target.isGpuActive(), | |
| backendName: target.getBackendName(), | |
| supportsVision: target.supportsVision(), | |
| supportsAudio: target.supportsAudio() | |
| }; | |
| } | |
| // js/src/worker_proxy.ts | |
| function createBridgeWorkerSource(moduleUrl) { | |
| return `import * as workerModule from ${JSON.stringify(moduleUrl)}; | |
| if (workerModule && typeof workerModule.enableBridgeWorkerHost === 'function') { workerModule.enableBridgeWorkerHost(); } | |
| `; | |
| } | |
| function resolveWorkerEntryUrl(moduleUrl) { | |
| if (typeof moduleUrl !== "string" || moduleUrl.length === 0) { | |
| return null; | |
| } | |
| try { | |
| const base = typeof window !== "undefined" && window.location?.href ? window.location.href : void 0; | |
| const url = new URL(moduleUrl, base); | |
| const path = url.pathname || ""; | |
| const usesDedicatedWorkerEntry = path.endsWith("_worker.js"); | |
| if (!usesDedicatedWorkerEntry) { | |
| url.searchParams.set(bridgeWorkerModeParam, "1"); | |
| } | |
| return url.toString(); | |
| } catch (_) { | |
| return null; | |
| } | |
| } | |
| function deriveBridgeModuleUrlFromWorkerEntry(moduleUrl) { | |
| if (typeof moduleUrl !== "string" || moduleUrl.length === 0) { | |
| return null; | |
| } | |
| try { | |
| const base = typeof window !== "undefined" && window.location?.href ? window.location.href : void 0; | |
| const url = new URL(moduleUrl, base); | |
| const path = url.pathname || ""; | |
| if (!path.endsWith("_worker.js")) { | |
| return null; | |
| } | |
| url.pathname = path.replace(/_worker\.js$/, ".js"); | |
| url.searchParams.delete(bridgeWorkerModeParam); | |
| return url.toString(); | |
| } catch (_) { | |
| return null; | |
| } | |
| } | |
| var BridgeWorkerProxy = class { | |
| constructor({ moduleUrl, config }) { | |
| this._config = config && typeof config === "object" ? config : {}; | |
| this._nextId = 1; | |
| this._pending = /* @__PURE__ */ new Map(); | |
| this._workerBlobUrl = null; | |
| let workerInitError = null; | |
| const moduleCandidates = [moduleUrl]; | |
| const bridgeModuleFallback = deriveBridgeModuleUrlFromWorkerEntry(moduleUrl); | |
| if (bridgeModuleFallback && bridgeModuleFallback !== moduleUrl) { | |
| moduleCandidates.push(bridgeModuleFallback); | |
| } | |
| for (const candidate of moduleCandidates) { | |
| if (this._worker) { | |
| break; | |
| } | |
| const directWorkerUrl = resolveWorkerEntryUrl(candidate); | |
| if (!directWorkerUrl) { | |
| continue; | |
| } | |
| try { | |
| this._worker = new Worker(directWorkerUrl, { type: "module" }); | |
| } catch (error) { | |
| workerInitError = error; | |
| } | |
| } | |
| for (const candidate of moduleCandidates) { | |
| if (this._worker) { | |
| break; | |
| } | |
| const source = createBridgeWorkerSource(candidate); | |
| this._workerBlobUrl = URL.createObjectURL( | |
| new Blob([source], { type: "text/javascript" }) | |
| ); | |
| try { | |
| this._worker = new Worker(this._workerBlobUrl, { type: "module" }); | |
| } catch (error) { | |
| workerInitError = error; | |
| URL.revokeObjectURL(this._workerBlobUrl); | |
| this._workerBlobUrl = null; | |
| } | |
| } | |
| if (!this._worker) { | |
| throw workerInitError || new Error("Failed to initialize bridge worker"); | |
| } | |
| this._ready = new Promise((resolve, reject) => { | |
| this._readyResolve = resolve; | |
| this._readyReject = reject; | |
| }); | |
| this._readyTimeoutHandle = null; | |
| this._armReadyTimeout(); | |
| this._worker.onmessage = (event) => { | |
| const message = event.data || {}; | |
| const type = message.type; | |
| if (type === "ready") { | |
| this._clearReadyTimeout(); | |
| this._readyResolve?.(); | |
| return; | |
| } | |
| const id = Number(message.id || 0); | |
| const pending = this._pending.get(id); | |
| if (!pending) { | |
| return; | |
| } | |
| if (type === "event") { | |
| pending.onEvent?.(message); | |
| return; | |
| } | |
| this._pending.delete(id); | |
| if (type === "error") { | |
| const errorMessage = String(message.message || "Worker request failed"); | |
| const workerError = message.embeddingErrorCode ? new EmbeddingError(message.embeddingErrorCode, errorMessage) : new Error(errorMessage); | |
| if (message.state && typeof message.state === "object") { | |
| workerError.state = message.state; | |
| } | |
| pending.reject(workerError); | |
| return; | |
| } | |
| pending.resolve(message); | |
| }; | |
| this._worker.onerror = (event) => { | |
| const message = event?.message || "Bridge worker crashed"; | |
| const error = new Error(String(message)); | |
| error.llamadartWorkerCrash = true; | |
| this._clearReadyTimeout(); | |
| this._readyReject?.(error); | |
| for (const pending of this._pending.values()) { | |
| pending.reject(error); | |
| } | |
| this._pending.clear(); | |
| }; | |
| this._worker.postMessage({ type: "init", config }); | |
| } | |
| async call(method, args, onEvent, transferList = [], operationMeta = null) { | |
| await this._ready; | |
| const id = this._nextId++; | |
| const timeoutMs = this._resolveRequestTimeoutMs(method, args); | |
| const transfers = Array.isArray(transferList) ? transferList.filter((item) => item != null) : []; | |
| return new Promise((resolve, reject) => { | |
| let timeoutHandle = null; | |
| const clearTimer = () => { | |
| if (timeoutHandle != null) { | |
| globalThis.clearTimeout(timeoutHandle); | |
| timeoutHandle = null; | |
| } | |
| }; | |
| const armTimer = () => { | |
| clearTimer(); | |
| if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { | |
| return; | |
| } | |
| timeoutHandle = globalThis.setTimeout(() => { | |
| this._pending.delete(id); | |
| reject(new Error(`Worker request timeout (${method}, ${timeoutMs}ms)`)); | |
| }, timeoutMs); | |
| }; | |
| armTimer(); | |
| this._pending.set(id, { | |
| resolve: (value) => { | |
| clearTimer(); | |
| resolve(value); | |
| }, | |
| reject: (error) => { | |
| clearTimer(); | |
| reject(error); | |
| }, | |
| onEvent: (event) => { | |
| armTimer(); | |
| onEvent?.(event); | |
| } | |
| }); | |
| const message = { type: "call", id, method, args }; | |
| if (operationMeta && typeof operationMeta === "object") { | |
| Object.assign(message, operationMeta); | |
| } | |
| this._worker.postMessage(message, transfers); | |
| }); | |
| } | |
| _resolveWorkerReadyTimeoutMs() { | |
| const configured = Number(this._config.workerInitTimeoutMs); | |
| if (!Number.isFinite(configured) || configured <= 0) { | |
| return 2e4; | |
| } | |
| return Math.max(3e3, Math.min(12e4, Math.trunc(configured))); | |
| } | |
| _clearReadyTimeout() { | |
| if (this._readyTimeoutHandle != null) { | |
| globalThis.clearTimeout(this._readyTimeoutHandle); | |
| this._readyTimeoutHandle = null; | |
| } | |
| } | |
| _armReadyTimeout() { | |
| this._clearReadyTimeout(); | |
| const timeoutMs = this._resolveWorkerReadyTimeoutMs(); | |
| this._readyTimeoutHandle = globalThis.setTimeout(() => { | |
| this._readyTimeoutHandle = null; | |
| const timeoutError = new Error(`Bridge worker init timeout (${timeoutMs}ms)`); | |
| this._readyReject?.(timeoutError); | |
| try { | |
| this._worker?.terminate(); | |
| } catch (_) { | |
| } | |
| }, timeoutMs); | |
| } | |
| _resolveRequestTimeoutMs(method, args = []) { | |
| const explicitGlobal = Number(this._config.workerRequestTimeoutMs); | |
| const clamp = (value, fallback) => { | |
| if (!Number.isFinite(value) || value <= 0) { | |
| return fallback; | |
| } | |
| return Math.max(5e3, Math.min(36e5, Math.trunc(value))); | |
| }; | |
| if (method === "loadModelFromUrl" || method === "loadDraftModel") { | |
| return clamp(Number(this._config.workerModelLoadTimeoutMs), clamp(explicitGlobal, 3 * 60 * 1e3)); | |
| } | |
| if (method === "loadMultimodalProjector") { | |
| return clamp(Number(this._config.workerMmprojLoadTimeoutMs), clamp(explicitGlobal, 8 * 60 * 1e3)); | |
| } | |
| if (method === "createCompletion") { | |
| return clamp(Number(this._config.workerCompletionTimeoutMs), clamp(explicitGlobal, 6 * 60 * 1e3)); | |
| } | |
| if (method === "synthesizeSpeech") { | |
| return clamp(Number(this._config.workerTextToSpeechTimeoutMs), clamp(explicitGlobal, 20 * 60 * 1e3)); | |
| } | |
| if (method === "loadDecisionHead" || method === "loadLoraAdapter") { | |
| return clamp(explicitGlobal, 10 * 60 * 1e3); | |
| } | |
| if (method === "runDecision") { | |
| const sequences = Array.isArray(args?.[1]) ? args[1].length : 0; | |
| return clamp(explicitGlobal, 10 * 60 * 1e3 + sequences * DECISION_WORKER_TIMEOUT_PER_SEQUENCE_MS); | |
| } | |
| return clamp(explicitGlobal, 12e4); | |
| } | |
| async dispose() { | |
| this._clearReadyTimeout(); | |
| let didTimeout = false; | |
| try { | |
| await Promise.race([ | |
| this.call("dispose", []), | |
| new Promise((resolve) => { | |
| globalThis.setTimeout(() => { | |
| didTimeout = true; | |
| resolve(null); | |
| }, 800); | |
| }) | |
| ]); | |
| } catch (_) { | |
| } | |
| if (didTimeout) { | |
| } | |
| for (const pending of this._pending.values()) { | |
| pending.reject(new Error("Bridge worker disposed")); | |
| } | |
| this._pending.clear(); | |
| this._worker.terminate(); | |
| if (this._workerBlobUrl) { | |
| URL.revokeObjectURL(this._workerBlobUrl); | |
| this._workerBlobUrl = null; | |
| } | |
| } | |
| }; | |
| // js/src/bridge.ts | |
| var LlamaWebGpuBridge = class { | |
| static supportsSafariAdaptiveGpu = LlamaWebGpuBridgeRuntime.supportsSafariAdaptiveGpu === true; | |
| static supportsCompletionUsage = true; | |
| constructor(config = {}) { | |
| this._config = config; | |
| this._runtime = null; | |
| this._workerProxy = null; | |
| this._workerGeneration = 0; | |
| this._workerDisposePromise = null; | |
| this._retiringWorkerDisposals = /* @__PURE__ */ new Set(); | |
| this._retiredWorkerProxies = /* @__PURE__ */ new WeakSet(); | |
| this._workerFallbackReason = null; | |
| this._metadata = {}; | |
| this._contextSize = 0; | |
| this._gpuActive = false; | |
| this._backendName = "WASM (Prototype bridge)"; | |
| this._supportsVision = false; | |
| this._supportsAudio = false; | |
| this._loadedModelUrl = null; | |
| this._loadedModelOptions = null; | |
| this._loadedMmProjUrl = null; | |
| this._loadedDraftModel = null; | |
| this._multimodalWorkerCpuMode = false; | |
| this._workerModelMissing = false; | |
| this._bridgeWarnRecent = /* @__PURE__ */ new Map(); | |
| this._operationQueueTail = null; | |
| this._activeOperation = null; | |
| this._nextOperationId = 0; | |
| this._lifecycleState = "open"; | |
| this._shadowStateTransactionDepth = 0; | |
| this._deferredShadowState = null; | |
| this._disposed = false; | |
| this._disposePromise = null; | |
| this._disposalWaiters = /* @__PURE__ */ new Set(); | |
| this._decisionHeads = /* @__PURE__ */ new Map(); | |
| this._nextDecisionHandle = 1; | |
| this._loraAdapters = /* @__PURE__ */ new Map(); | |
| this._activeLoraScales = /* @__PURE__ */ new Map(); | |
| this._nextLoraHandle = 1; | |
| if (this._shouldUseWorker()) { | |
| try { | |
| this._workerProxy = this._createWorkerProxy(); | |
| } catch (error) { | |
| this._disableWorkerFallback(error); | |
| } | |
| } | |
| if (this._workerProxy) { | |
| this._workerGeneration = 1; | |
| } | |
| if (!this._workerProxy) { | |
| this._runtime = this._createRuntime(); | |
| this._captureDirectRuntimeState(); | |
| } | |
| } | |
| _createRuntime() { | |
| return new LlamaWebGpuBridgeRuntime({ | |
| ...this._config, | |
| disableWorker: true | |
| }); | |
| } | |
| _operationSignal(operation = this._activeOperation) { | |
| return operation?.abortController?.signal || null; | |
| } | |
| _recordWorkerGeneration(operation = this._activeOperation, proxy = this._workerProxy) { | |
| if (!operation) { | |
| return; | |
| } | |
| const generation = this._workerGeneration || 0; | |
| operation.workerGeneration = generation; | |
| operation.workerGenerations?.add(generation); | |
| if (proxy) { | |
| operation.workerProxy = proxy; | |
| operation.currentWorkerProxy = proxy; | |
| operation.workerProxies?.add(proxy); | |
| } | |
| } | |
| _advanceWorkerGeneration() { | |
| this._workerGeneration = (this._workerGeneration || 0) + 1; | |
| this._recordWorkerGeneration(); | |
| return this._workerGeneration; | |
| } | |
| _isOperationCurrent(operation) { | |
| return !!(operation && this._activeOperation === operation && this._lifecycleState !== "disposed" && operation.state !== "cancelling" && operation.state !== "cancelled" && operation.state !== "disposed"); | |
| } | |
| _isOperationCallbackCurrent(operation, proxy) { | |
| return this._isOperationCurrent(operation) && proxy === this._workerProxy; | |
| } | |
| _sendWorkerCancel(proxy) { | |
| if (!proxy) { | |
| return; | |
| } | |
| if (typeof proxy.call !== "function") { | |
| if (proxy === this._workerProxy && typeof this._callWorker === "function") { | |
| try { | |
| Promise.resolve(this._callWorker("cancel", [])).catch(() => { | |
| }); | |
| } catch (_) { | |
| } | |
| } | |
| return; | |
| } | |
| try { | |
| Promise.resolve(proxy.call("cancel", [], null)).catch(() => { | |
| }); | |
| } catch (_) { | |
| } | |
| } | |
| _cancelOperation(operation, reason = "user") { | |
| if (!operation) { | |
| return; | |
| } | |
| if (reason === "worker-timeout") { | |
| if (operation.cancelRequested || operation.workerTimeoutCancellationSent) { | |
| return; | |
| } | |
| operation.workerTimeoutCancellationSent = true; | |
| operation.cancelReason = reason; | |
| operation.state = "recovering"; | |
| const proxies2 = new Set(operation.workerProxies || []); | |
| if (operation.workerProxy) { | |
| proxies2.add(operation.workerProxy); | |
| } | |
| for (const proxy of proxies2) { | |
| this._sendWorkerCancel(proxy); | |
| } | |
| return; | |
| } | |
| if (operation.cancelRequested) { | |
| return; | |
| } | |
| operation.cancelRequested = true; | |
| operation.cancelReason = reason; | |
| operation.state = "cancelling"; | |
| try { | |
| operation.abortController?.abort(); | |
| } catch (_) { | |
| } | |
| const proxies = new Set(operation.workerProxies || []); | |
| if (operation.workerProxy) { | |
| proxies.add(operation.workerProxy); | |
| } | |
| for (const proxy of proxies) { | |
| this._sendWorkerCancel(proxy); | |
| } | |
| const runtimes = new Set(operation.runtimes || []); | |
| if (operation.runtime) { | |
| runtimes.add(operation.runtime); | |
| } | |
| for (const runtime of runtimes) { | |
| try { | |
| runtime?.cancel?.(); | |
| } catch (_) { | |
| } | |
| } | |
| } | |
| _isOperationCancellation(error, operation = this._activeOperation) { | |
| if (operation?.callerSignal?.aborted) { | |
| return true; | |
| } | |
| if (operation?.cancelRequested && operation.cancelReason !== "worker-timeout") { | |
| return true; | |
| } | |
| return error?.name === "AbortError" && !this._isWorkerTimeoutError(error); | |
| } | |
| _throwIfOperationCancelled(error, message = "Bridge operation was cancelled.", operation = this._activeOperation) { | |
| if (this._isOperationCancellation(error, operation)) { | |
| throw createAbortError(message); | |
| } | |
| } | |
| /** | |
| * Guards the success path: a worker that ignored cancellation and settled | |
| * with a full result must still reject once the caller's signal was aborted, | |
| * and must not commit that result. Only the caller's signal counts here. | |
| * A bare `cancel()` stays the documented best-effort control signal, so a | |
| * result that still arrived is delivered rather than discarded. | |
| */ | |
| _throwIfCallerCancelled(message = "Bridge operation was cancelled.", operation = this._activeOperation) { | |
| if (operation?.callerSignal?.aborted) { | |
| throw createAbortError(message); | |
| } | |
| } | |
| _retireWorkerProxy(proxy) { | |
| if (!proxy || typeof proxy !== "object") { | |
| return Promise.resolve(); | |
| } | |
| if (this._retiredWorkerProxies?.has(proxy)) { | |
| return Promise.resolve(); | |
| } | |
| this._retiredWorkerProxies?.add(proxy); | |
| const operation = this._activeOperation; | |
| operation?.workerProxies?.add(proxy); | |
| let disposal; | |
| try { | |
| disposal = Promise.resolve(proxy.dispose?.()).catch(() => { | |
| }); | |
| } catch (_) { | |
| disposal = Promise.resolve(); | |
| } | |
| this._retiringWorkerDisposals ??= /* @__PURE__ */ new Set(); | |
| this._retiringWorkerDisposals.add(disposal); | |
| this._workerDisposePromise = disposal; | |
| operation?.retirements?.add(disposal); | |
| disposal.then( | |
| () => { | |
| this._retiringWorkerDisposals.delete(disposal); | |
| if (this._workerDisposePromise === disposal && this._retiringWorkerDisposals.size === 0) { | |
| this._workerDisposePromise = null; | |
| } | |
| }, | |
| () => { | |
| this._retiringWorkerDisposals.delete(disposal); | |
| if (this._workerDisposePromise === disposal && this._retiringWorkerDisposals.size === 0) { | |
| this._workerDisposePromise = null; | |
| } | |
| } | |
| ); | |
| return disposal; | |
| } | |
| async _waitForWorkerDisposal() { | |
| this._retiringWorkerDisposals ??= /* @__PURE__ */ new Set(); | |
| while (this._retiringWorkerDisposals.size > 0) { | |
| const pending = [...this._retiringWorkerDisposals]; | |
| await Promise.all(pending); | |
| } | |
| this._workerDisposePromise = null; | |
| } | |
| /** | |
| * llama.cpp exposes one model/context per bridge instance and publishes | |
| * results through process-global C buffers (`last_output`, `last_tokens_json`, | |
| * `last_detokenized`, `last_embedding_json`, ...). Every async call that reads | |
| * or mutates that runtime therefore has to own it exclusively for its whole | |
| * duration: a second caller interleaving between the work ccall and the buffer | |
| * read would otherwise observe the other caller's result. This FIFO queue is | |
| * the single-writer lock, shared by the worker and direct-runtime paths. | |
| * | |
| * A caller whose signal is already aborted, or that aborts while still queued, | |
| * never runs and never dispatches; its slot is skipped in turn without | |
| * touching the unrelated operation that currently owns the runtime. Once an | |
| * operation has started, its signal is routed to `_cancelOperation` so the | |
| * cancel reaches the worker generation and runtime that operation captured | |
| * rather than whatever topology happens to be current. | |
| */ | |
| _runExclusive(run, options = {}) { | |
| const signal = options.signal || null; | |
| const abortMessage = options.abortMessage || "Bridge operation was cancelled."; | |
| const allowDisposed = options.allowDisposed === true; | |
| const abortError = () => createAbortError(abortMessage); | |
| const lifecycleState = this._lifecycleState || (this._disposed ? "disposed" : "open"); | |
| if (!allowDisposed && (this._disposed || lifecycleState !== "open")) { | |
| return Promise.reject(new Error(BRIDGE_DISPOSED_MESSAGE)); | |
| } | |
| if (signal?.aborted) { | |
| return Promise.reject(abortError()); | |
| } | |
| const operation = { | |
| id: ++this._nextOperationId, | |
| kind: options.kind || "bridge-operation", | |
| state: "queued", | |
| callerSignal: signal, | |
| abortController: typeof AbortController === "function" ? new AbortController() : null, | |
| cancelRequested: false, | |
| cancelReason: null, | |
| workerTimeoutCancellationSent: false, | |
| workerGeneration: this._workerGeneration || 0, | |
| workerGenerations: /* @__PURE__ */ new Set([this._workerGeneration || 0]), | |
| workerProxy: this._workerProxy || null, | |
| runtime: this._runtime || null, | |
| workerProxies: new Set(this._workerProxy ? [this._workerProxy] : []), | |
| runtimes: new Set(this._runtime ? [this._runtime] : []), | |
| modelSource: null, | |
| modelOptions: null, | |
| projectorSource: null, | |
| recoveryAttempted: false, | |
| generationStarted: false, | |
| retirements: /* @__PURE__ */ new Set(), | |
| deferredShadowState: null | |
| }; | |
| const predecessor = this._operationQueueTail; | |
| let releaseSuccessor = () => { | |
| }; | |
| const queueSlot = new Promise((resolve) => { | |
| releaseSuccessor = () => resolve(void 0); | |
| }); | |
| this._operationQueueTail = queueSlot; | |
| let started = false; | |
| let removeAbortListener = null; | |
| let removeDisposalWaiter = null; | |
| const detachWatchers = () => { | |
| removeAbortListener?.(); | |
| removeDisposalWaiter?.(); | |
| removeAbortListener = null; | |
| removeDisposalWaiter = null; | |
| }; | |
| const watchesAbort = signal != null && typeof signal.addEventListener === "function"; | |
| const watchesDisposal = !allowDisposed; | |
| const ordered = (async () => { | |
| if (predecessor) { | |
| await predecessor; | |
| } | |
| if (signal?.aborted) { | |
| operation.state = "skipped"; | |
| throw abortError(); | |
| } | |
| if (!allowDisposed && (this._disposed || this._lifecycleState === "disposing")) { | |
| operation.state = "disposed"; | |
| throw new Error(BRIDGE_DISPOSED_MESSAGE); | |
| } | |
| started = true; | |
| removeDisposalWaiter?.(); | |
| removeDisposalWaiter = null; | |
| operation.state = "starting"; | |
| operation.workerProxy = this._workerProxy || null; | |
| operation.workerGeneration = this._workerGeneration || 0; | |
| operation.workerGenerations.add(operation.workerGeneration); | |
| operation.runtime = this._runtime || null; | |
| if (operation.workerProxy) { | |
| operation.workerProxies.add(operation.workerProxy); | |
| operation.currentWorkerProxy = operation.workerProxy; | |
| } | |
| if (operation.runtime) { | |
| operation.runtimes.add(operation.runtime); | |
| } | |
| this._activeOperation = operation; | |
| operation.state = "running"; | |
| let operationFailed = false; | |
| try { | |
| return await run(operation); | |
| } catch (error) { | |
| operationFailed = true; | |
| throw error; | |
| } finally { | |
| let terminalState = operation.cancelRequested ? "cancelled" : operationFailed ? "failed" : "completed"; | |
| operation.state = terminalState === "completed" ? "settling" : terminalState; | |
| try { | |
| this._captureDirectRuntimeState(); | |
| } catch (error) { | |
| terminalState = "failed"; | |
| throw error; | |
| } finally { | |
| if (this._activeOperation === operation) { | |
| this._activeOperation = null; | |
| } | |
| operation.state = terminalState; | |
| } | |
| } | |
| })(); | |
| const release = () => { | |
| if (this._operationQueueTail === queueSlot) { | |
| this._operationQueueTail = null; | |
| } | |
| releaseSuccessor(); | |
| }; | |
| ordered.then(release, release); | |
| if (!watchesAbort && !watchesDisposal) { | |
| return ordered; | |
| } | |
| const abandonedWhileQueued = new Promise((_, reject) => { | |
| if (watchesAbort) { | |
| const onAbort = () => { | |
| if (!started) { | |
| operation.state = "skipped"; | |
| reject(abortError()); | |
| return; | |
| } | |
| this._cancelOperation(operation, "signal"); | |
| }; | |
| signal.addEventListener("abort", onAbort, { once: true }); | |
| removeAbortListener = () => signal.removeEventListener("abort", onAbort); | |
| if (signal.aborted) { | |
| onAbort(); | |
| } | |
| } | |
| if (watchesDisposal && !started) { | |
| removeDisposalWaiter = this._addDisposalWaiter(() => { | |
| if (!started) { | |
| operation.state = "disposed"; | |
| reject(new Error(BRIDGE_DISPOSED_MESSAGE)); | |
| } | |
| }); | |
| } | |
| }); | |
| const raced = Promise.race([ordered, abandonedWhileQueued]); | |
| raced.then(detachWatchers, detachWatchers); | |
| return raced; | |
| } | |
| /** | |
| * Registers a disposal listener for the lifetime of one pending operation and | |
| * returns its remover. A listener set is used rather than a long-lived promise | |
| * because a promise reaction cannot be detached. | |
| */ | |
| _addDisposalWaiter(onDisposed) { | |
| const waiters = this._disposalWaiters; | |
| waiters.add(onDisposed); | |
| return () => { | |
| waiters.delete(onDisposed); | |
| }; | |
| } | |
| /** | |
| * True while `operation` is the non-dispose owner of the queue slot. Teardown | |
| * is queued behind it, so it is allowed to finish the work it already started | |
| * even once disposal has been requested. | |
| */ | |
| _isSettleFirstOwner(operation = this._activeOperation) { | |
| return !!(operation && this._activeOperation === operation && operation.kind !== "dispose" && operation.state !== "cancelled" && operation.state !== "disposed"); | |
| } | |
| /** | |
| * Lifecycle gate for the helpers that stay off the queue. They must not | |
| * dispatch to a torn-down worker, and must not lazily recreate the runtime | |
| * disposal just released. `disposing` still admits the operation that already | |
| * owns the slot, because disposal settles it first rather than interrupting | |
| * it; once teardown has run, every caller is rejected. | |
| */ | |
| _throwIfDisposed() { | |
| if (this._lifecycleState === "disposed") { | |
| throw new Error(BRIDGE_DISPOSED_MESSAGE); | |
| } | |
| if ((this._disposed || this._lifecycleState === "disposing") && !this._isSettleFirstOwner()) { | |
| throw new Error(BRIDGE_DISPOSED_MESSAGE); | |
| } | |
| } | |
| _notifyDisposalWaiters() { | |
| const waiters = this._disposalWaiters; | |
| if (waiters.size === 0) { | |
| return; | |
| } | |
| this._disposalWaiters = /* @__PURE__ */ new Set(); | |
| for (const waiter of waiters) { | |
| waiter(); | |
| } | |
| } | |
| /** | |
| * Mirror the direct runtime's model/context state into facade fields so the | |
| * synchronous getters can answer from cache. Reading them straight from the | |
| * runtime would ccall into the singleton core outside the queue, racing | |
| * whichever async operation currently owns it. | |
| */ | |
| _captureDirectRuntimeState() { | |
| if (this._workerProxy || !this._runtime) { | |
| return; | |
| } | |
| try { | |
| this._metadata = this._runtime.getModelMetadata(); | |
| this._contextSize = this._runtime.getContextSize(); | |
| this._gpuActive = this._runtime.isGpuActive(); | |
| this._backendName = this._runtime.getBackendName(); | |
| this._supportsVision = this._runtime.supportsVision(); | |
| this._supportsAudio = this._runtime.supportsAudio(); | |
| } catch (_) { | |
| } | |
| } | |
| /** | |
| * prefetch/evict stay off the queue because they only touch Cache Storage and | |
| * the network. In direct mode their plain-JS cache bookkeeping is copied into | |
| * the facade snapshot. A worker-backed snapshot remains wholly worker-owned so | |
| * a separate cache-only runtime cannot misdescribe the active worker model. | |
| */ | |
| _refreshCacheMetadataSnapshot() { | |
| const runtime = this._runtime; | |
| if (this._workerProxy || !runtime) { | |
| return; | |
| } | |
| this._metadata = { | |
| ...this._metadata || {}, | |
| "llamadart.webgpu.model_source": String(runtime._modelSource ?? ""), | |
| "llamadart.webgpu.model_cache_state": String(runtime._modelCacheState ?? ""), | |
| "llamadart.webgpu.model_cache_name": String(runtime._modelCacheName ?? ""), | |
| "llamadart.webgpu.runtime_notes": Array.isArray(runtime._runtimeNotes) ? runtime._runtimeNotes.join(";") : "" | |
| }; | |
| } | |
| _sanitizeModelLoadOptions(options = {}) { | |
| const source = options && typeof options === "object" ? options : {}; | |
| const sanitized = { ...source }; | |
| delete sanitized.progressCallback; | |
| delete sanitized.signal; | |
| return sanitized; | |
| } | |
| _createCpuSafeMultimodalLoadOptions(options = {}) { | |
| const sanitized = this._sanitizeModelLoadOptions(options); | |
| sanitized.nGpuLayers = 0; | |
| if (Number.isFinite(Number(sanitized.nCtx)) && Number(sanitized.nCtx) > 4096) { | |
| sanitized.nCtx = 4096; | |
| } | |
| if (!Number.isFinite(Number(sanitized.nThreads)) || Number(sanitized.nThreads) <= 0) { | |
| sanitized.nThreads = 4; | |
| } else { | |
| sanitized.nThreads = Math.min(4, Math.max(1, Math.trunc(Number(sanitized.nThreads)))); | |
| } | |
| sanitized.nThreadsBatch = sanitized.nThreads; | |
| if (!Number.isFinite(Number(sanitized.nBatch)) || Number(sanitized.nBatch) <= 0) { | |
| sanitized.nBatch = 128; | |
| } else { | |
| sanitized.nBatch = Math.min(128, Math.max(32, Math.trunc(Number(sanitized.nBatch)))); | |
| } | |
| if (!Number.isFinite(Number(sanitized.nUbatch)) || Number(sanitized.nUbatch) <= 0) { | |
| sanitized.nUbatch = Math.min(64, sanitized.nBatch); | |
| } else { | |
| sanitized.nUbatch = Math.min( | |
| sanitized.nBatch, | |
| Math.min(64, Math.max(1, Math.trunc(Number(sanitized.nUbatch)))) | |
| ); | |
| } | |
| return sanitized; | |
| } | |
| _rememberLoadedModel(url, options = {}) { | |
| const normalizedUrl = cloneModelSource(url); | |
| if (!hasModelSource(normalizedUrl)) { | |
| return; | |
| } | |
| this._loadedModelUrl = normalizedUrl; | |
| this._loadedModelOptions = this._sanitizeModelLoadOptions(options); | |
| this._loadedMmProjUrl = null; | |
| this._forgetLoraAdapters(); | |
| this._loadedDraftModel = null; | |
| this._multimodalWorkerCpuMode = this._workerProxy != null; | |
| this._workerModelMissing = false; | |
| if (this._activeOperation) { | |
| this._activeOperation.modelSource = cloneModelSource(normalizedUrl); | |
| this._activeOperation.modelOptions = { ...this._loadedModelOptions }; | |
| this._activeOperation.projectorSource = null; | |
| } | |
| } | |
| _forgetLoadedModel() { | |
| this._loadedModelUrl = null; | |
| this._loadedModelOptions = null; | |
| this._loadedMmProjUrl = null; | |
| this._forgetLoraAdapters(); | |
| this._loadedDraftModel = null; | |
| this._multimodalWorkerCpuMode = false; | |
| this._workerModelMissing = false; | |
| } | |
| /** | |
| * A failed load can leave its target without any model: the runtime drops | |
| * the previous model once the new download starts. Keep the facade's model | |
| * source only while the target still reports a loaded model, so recovery | |
| * never reloads a model the target no longer holds. | |
| * | |
| * A worker's reply is read from `attempt`, not the facade snapshot: a | |
| * cancelled operation no longer accepts worker state, so the snapshot can | |
| * still describe a model the worker already dropped. Only the bookkeeping is | |
| * corrected here; a cancelled load never publishes the worker's snapshot. | |
| */ | |
| _syncLoadedModelAfterFailedLoad(attempt) { | |
| const proxy = this._workerProxy; | |
| if (!proxy) { | |
| if (!(Number(this._runtime?._modelBytes) > 0)) { | |
| this._forgetLoadedModel(); | |
| } | |
| return; | |
| } | |
| const fromCurrentWorker = attempt.proxy === proxy; | |
| const replacedModel = fromCurrentWorker && attempt.loaded; | |
| const state = fromCurrentWorker ? attempt.state : null; | |
| const modelBytes = Number( | |
| (state ? state.metadata : this._metadata)?.["llamadart.webgpu.model_bytes"] | |
| ); | |
| if (replacedModel || !(modelBytes > 0)) { | |
| this._forgetLoadedModel(); | |
| } | |
| } | |
| _rememberLoadedMmProj(url) { | |
| const normalizedUrl = String(url || "").trim(); | |
| if (normalizedUrl.length === 0) { | |
| return; | |
| } | |
| this._loadedMmProjUrl = normalizedUrl; | |
| if (this._activeOperation) { | |
| this._activeOperation.projectorSource = normalizedUrl; | |
| } | |
| } | |
| _hasMediaParts(options = {}) { | |
| return Array.isArray(options?.parts) && options.parts.length > 0; | |
| } | |
| async _replaceWorkerProxyForMultimodalCpuMode() { | |
| this._throwIfDisposed(); | |
| const hadWorkerProxy = this._workerProxy != null; | |
| if (hadWorkerProxy) { | |
| const staleProxy = this._workerProxy; | |
| this._advanceWorkerGeneration(); | |
| this._workerProxy = null; | |
| this._retireWorkerProxy(staleProxy); | |
| await this._waitForWorkerDisposal(); | |
| this._throwIfDisposed(); | |
| } | |
| let replacement; | |
| try { | |
| replacement = this._createWorkerProxy(); | |
| } catch (error) { | |
| this._disableWorkerFallback(error); | |
| throw error; | |
| } | |
| try { | |
| this._throwIfDisposed(); | |
| } catch (error) { | |
| this._retireWorkerProxy(replacement); | |
| throw error; | |
| } | |
| if (!hadWorkerProxy) { | |
| this._advanceWorkerGeneration(); | |
| } | |
| this._workerProxy = replacement; | |
| this._recordWorkerGeneration(this._activeOperation, replacement); | |
| this._multimodalWorkerCpuMode = false; | |
| this._workerModelMissing = true; | |
| } | |
| _isRecoverableWorkerFsError(error) { | |
| const text = serializeWorkerError(error).toLowerCase(); | |
| if (text.includes("model shard:")) { | |
| return false; | |
| } | |
| return text.includes("fs error") || text.includes("no such file") || text.includes("not found") || text.includes("invalid argument") || text.includes("timed out"); | |
| } | |
| _isWorkerRequestTimeoutError(error) { | |
| const text = serializeWorkerError(error).toLowerCase(); | |
| return text.includes("worker request timeout") || text.includes("worker init timeout") || text.includes("worker timed out"); | |
| } | |
| async _ensureWorkerMultimodalCpuMode() { | |
| this._throwIfDisposed(); | |
| if (!this._workerProxy) { | |
| await this._replaceWorkerProxyForMultimodalCpuMode(); | |
| } | |
| this._throwIfDisposed(); | |
| if (this._multimodalWorkerCpuMode) { | |
| return true; | |
| } | |
| if (!hasModelSource(this._loadedModelUrl)) { | |
| return false; | |
| } | |
| const selectedOptions = this._sanitizeModelLoadOptions( | |
| this._loadedModelOptions || {} | |
| ); | |
| const operation = this._activeOperation; | |
| if (operation) { | |
| operation.recoveryAttempted = true; | |
| operation.state = "recovering"; | |
| } | |
| const previousDeferredState = this._deferredShadowState; | |
| this._shadowStateTransactionDepth += 1; | |
| this._deferredShadowState = null; | |
| let recoverySucceeded = false; | |
| const applyWorkerSafeMode = () => this._loadRememberedModelIntoWorker(selectedOptions); | |
| try { | |
| await applyWorkerSafeMode(); | |
| recoverySucceeded = true; | |
| this._emitBridgeWarn( | |
| "llamadart: multimodal worker prepared in selected backend mode." | |
| ); | |
| return true; | |
| } catch (error) { | |
| this._throwIfDisposed(); | |
| this._emitBridgeWarn( | |
| `llamadart: multimodal worker setup failed once; restarting worker (${serializeWorkerError(error)}).` | |
| ); | |
| await this._replaceWorkerProxyForMultimodalCpuMode(); | |
| await applyWorkerSafeMode(); | |
| recoverySucceeded = true; | |
| this._emitBridgeWarn( | |
| "llamadart: multimodal worker recovered after restart." | |
| ); | |
| return true; | |
| } finally { | |
| this._shadowStateTransactionDepth = Math.max(0, this._shadowStateTransactionDepth - 1); | |
| if (this._shadowStateTransactionDepth === 0) { | |
| if (recoverySucceeded && this._deferredShadowState && this._workerProxy) { | |
| this._applyShadowStateFrom(this._workerProxy, this._deferredShadowState); | |
| } | |
| this._deferredShadowState = previousDeferredState || null; | |
| } | |
| if (operation && this._activeOperation === operation && operation.state === "recovering") { | |
| operation.state = operation.cancelRequested ? "cancelling" : "running"; | |
| } | |
| } | |
| } | |
| async _loadRememberedModelIntoWorker(selectedOptions) { | |
| this._throwIfDisposed(); | |
| this._dropLoraAdaptersOf(this._workerProxy); | |
| await this._callWorker("loadModelFromUrl", [this._loadedModelUrl, selectedOptions]); | |
| this._throwIfDisposed(); | |
| if (typeof this._loadedMmProjUrl === "string" && this._loadedMmProjUrl.length > 0) { | |
| await this._callWorker("loadMultimodalProjector", [this._loadedMmProjUrl]); | |
| } | |
| const draft = this._loadedDraftModel; | |
| if (draft) { | |
| try { | |
| await this._callWorker("loadDraftModel", [draft.url, draft.options]); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Draft model reload was cancelled."); | |
| this._forgetDraftModelAfterReloadFailure(error); | |
| } | |
| } | |
| this._throwIfDisposed(); | |
| await this._restoreLoraAdapters(); | |
| this._throwIfDisposed(); | |
| this._workerModelMissing = false; | |
| this._loadedModelOptions = selectedOptions; | |
| this._multimodalWorkerCpuMode = true; | |
| } | |
| /** | |
| * Multimodal recovery can replace the worker and then fail to reload the | |
| * model, its projector or its applied LoRA adapters into it. The facade | |
| * still holds that model, but the fresh worker would answer every request | |
| * with "No model loaded" or without the adapters. Reload them first, and | |
| * again on the next call if this fails. A failure here reaches the caller's | |
| * own worker-error handling. | |
| */ | |
| async _restoreWorkerModelIfMissing() { | |
| if (!this._workerProxy || this._workerModelMissing !== true || !hasModelSource(this._loadedModelUrl)) { | |
| return; | |
| } | |
| this._emitBridgeWarn( | |
| "llamadart: bridge worker was restarted without its model; reloading it." | |
| ); | |
| await this._loadRememberedModelIntoWorker( | |
| this._sanitizeModelLoadOptions(this._loadedModelOptions || {}) | |
| ); | |
| } | |
| _isDispatchWorkgroupLimitError(error) { | |
| const text = serializeWorkerError(error).toLowerCase(); | |
| return text.includes("dispatch workgroup count") || text.includes("max compute workgroups per dimension") || text.includes("invalid commandbuffer") || text.includes("ggml_webgpu: device error") || text.includes("runtimeerror: aborted") || text.includes("aborted()"); | |
| } | |
| // An aborted or trapped Wasm core stays dead. The worker host posts only | |
| // the message text, so a worker-side RuntimeError arrives without its name. | |
| _isWasmCoreAbortError(error) { | |
| if (typeof WebAssembly !== "undefined" && error instanceof WebAssembly.RuntimeError) { | |
| return true; | |
| } | |
| const text = serializeWorkerError(error).toLowerCase(); | |
| return text.includes("aborted(") || text.includes("runtimeerror") || text.includes("unreachable") || text.includes("memory access out of bounds") || text.includes("function signature mismatch") || text.includes("program terminated with exit"); | |
| } | |
| // A worker request falls back only when the worker can no longer serve | |
| // requests: it crashed, stalled, timed out, never initialized, or its core | |
| // aborted. A core error from a healthy worker (invalid grammar, a context | |
| // limit, an unsupported template, a model without embeddings) is | |
| // deterministic; retrying on the main thread fails the same way and strands | |
| // the session there. | |
| _isWorkerUnusableError(error) { | |
| if (error && typeof error === "object" && error.llamadartWorkerCrash === true) { | |
| return true; | |
| } | |
| return this._isWorkerTimeoutError(error) || this._isWasmCoreAbortError(error) || this._shouldFallbackToMainThread(error); | |
| } | |
| _isWorkerTimeoutError(error) { | |
| if (error && typeof error === "object" && error.llamadartWorkerTimeout === true) { | |
| return true; | |
| } | |
| const text = serializeWorkerError(error).toLowerCase(); | |
| return text.includes("worker completion stalled") || text.includes("worker createcompletion stalled") || text.includes("worker timed out") || text.includes("worker timeout"); | |
| } | |
| _isCpuModelMode() { | |
| const requestedLayers = Number(this._loadedModelOptions?.nGpuLayers); | |
| if (Number.isFinite(requestedLayers)) { | |
| return requestedLayers === 0; | |
| } | |
| const metadataLayers = Number(this._metadata?.["llamadart.webgpu.n_gpu_layers"]); | |
| if (Number.isFinite(metadataLayers)) { | |
| return metadataLayers === 0; | |
| } | |
| return false; | |
| } | |
| _workerCompletionStallTimeoutMs(options = {}) { | |
| const configured = Number(this._config?.workerGenerationStallTimeoutMs); | |
| if (Number.isFinite(configured) && configured > 0) { | |
| return Math.max(5e3, Math.min(3e5, Math.trunc(configured))); | |
| } | |
| if (!this._hasMediaParts(options)) { | |
| return 9e4; | |
| } | |
| if (this._isCpuModelMode()) { | |
| return 0; | |
| } | |
| return 18e4; | |
| } | |
| async _ensureRuntimeReadyAfterWorkerFallback(options = {}, fallbackError = null) { | |
| this._throwIfDisposed(); | |
| await this._waitForWorkerDisposal(); | |
| this._throwIfDisposed(); | |
| if (!this._runtime) { | |
| this._runtime = this._createRuntime(); | |
| } | |
| const operation = this._activeOperation; | |
| operation?.runtimes?.add(this._runtime); | |
| if (operation && operation.state === "running") { | |
| operation.state = "recovering"; | |
| } | |
| const forceReloadRequested = options?._llamadartForceRuntimeReload === true; | |
| const mediaPartsRequested = this._hasMediaParts(options); | |
| const textToSpeechRequested = options?._llamadartTextToSpeech === true; | |
| const multimodalRuntimeRequired = mediaPartsRequested || textToSpeechRequested; | |
| const shouldEnsureMultimodalInRuntime = multimodalRuntimeRequired && typeof this._loadedMmProjUrl === "string" && this._loadedMmProjUrl.length > 0; | |
| const workerTimedOut = this._isWorkerTimeoutError(fallbackError); | |
| const dispatchWorkgroupFallback = this._isDispatchWorkgroupLimitError(fallbackError); | |
| const loadedGpuLayers = Number(this._loadedModelOptions?.nGpuLayers); | |
| const metadataGpuLayers = Number(this._metadata?.["llamadart.webgpu.n_gpu_layers"]); | |
| const modelLoadedWithGpu = Number.isFinite(loadedGpuLayers) ? loadedGpuLayers !== 0 : Number.isFinite(metadataGpuLayers) ? metadataGpuLayers !== 0 : true; | |
| const shouldUseCpuMultimodalFallback = multimodalRuntimeRequired && modelLoadedWithGpu && (dispatchWorkgroupFallback || workerTimedOut); | |
| try { | |
| if (Number(this._runtime?._modelBytes) > 0 && !forceReloadRequested && !shouldUseCpuMultimodalFallback) { | |
| await this._ensureRuntimeDraftModel(); | |
| if (shouldEnsureMultimodalInRuntime) { | |
| const runtimeSupportsMedia = typeof this._runtime.supportsVision === "function" && this._runtime.supportsVision() || typeof this._runtime.supportsAudio === "function" && this._runtime.supportsAudio(); | |
| if (!runtimeSupportsMedia) { | |
| this._throwIfDisposed(); | |
| await this._runtime.loadMultimodalProjector(this._loadedMmProjUrl); | |
| this._throwIfDisposed(); | |
| if (Array.isArray(this._runtime._runtimeNotes)) { | |
| this._runtime._runtimeNotes.push("worker_fallback_reload_mmproj"); | |
| } | |
| } | |
| } | |
| await this._restoreLoraAdapters(); | |
| this._throwIfDisposed(); | |
| return; | |
| } | |
| if (!hasModelSource(this._loadedModelUrl)) { | |
| return; | |
| } | |
| const loadOptions = shouldUseCpuMultimodalFallback ? this._createCpuSafeMultimodalLoadOptions(this._loadedModelOptions || {}) : this._sanitizeModelLoadOptions(this._loadedModelOptions || {}); | |
| const recoverySignal = this._operationSignal(operation) || options.signal || null; | |
| if (recoverySignal) { | |
| loadOptions.signal = recoverySignal; | |
| } | |
| if (shouldUseCpuMultimodalFallback) { | |
| if (textToSpeechRequested) { | |
| this._emitBridgeWarn( | |
| "llamadart: retrying text-to-speech once with CPU fallback after WebGPU failure." | |
| ); | |
| } else if (workerTimedOut) { | |
| this._emitBridgeWarn( | |
| "llamadart: retrying multimodal generation with CPU fallback after worker timeout." | |
| ); | |
| } else { | |
| this._emitBridgeWarn( | |
| "llamadart: retrying multimodal generation with CPU fallback after WebGPU workgroup limit failure." | |
| ); | |
| } | |
| } | |
| if (workerTimedOut) { | |
| this._emitBridgeWarn( | |
| "llamadart: bridge worker completion stalled; restarting generation path on main-thread runtime." | |
| ); | |
| } | |
| this._throwIfDisposed(); | |
| this._dropLoraAdaptersOf(this._runtime); | |
| await this._runtime.loadModelFromUrl(this._loadedModelUrl, loadOptions); | |
| this._throwIfDisposed(); | |
| if (Array.isArray(this._runtime._runtimeNotes)) { | |
| this._runtime._runtimeNotes.push("worker_fallback_reload_model"); | |
| if (forceReloadRequested) { | |
| this._runtime._runtimeNotes.push("worker_fallback_reload_forced"); | |
| } | |
| if (workerTimedOut) { | |
| this._runtime._runtimeNotes.push("worker_fallback_timeout"); | |
| } | |
| if (shouldUseCpuMultimodalFallback) { | |
| this._runtime._runtimeNotes.push("worker_fallback_cpu_multimodal"); | |
| if (textToSpeechRequested) { | |
| this._runtime._runtimeNotes.push("worker_fallback_cpu_text_to_speech"); | |
| } | |
| } | |
| } | |
| if (shouldEnsureMultimodalInRuntime) { | |
| this._throwIfDisposed(); | |
| await this._runtime.loadMultimodalProjector(this._loadedMmProjUrl); | |
| this._throwIfDisposed(); | |
| } | |
| await this._restoreLoraAdapters(); | |
| this._throwIfDisposed(); | |
| await this._ensureRuntimeDraftModel(); | |
| } catch (error) { | |
| if (this._disposed || this._lifecycleState === "disposing") { | |
| throw new Error(BRIDGE_DISPOSED_MESSAGE); | |
| } | |
| if (error?.name === "AbortError") { | |
| throw error; | |
| } | |
| try { | |
| await this._runtime?.dispose?.(); | |
| } catch (_) { | |
| } | |
| this._runtime = null; | |
| this._metadata = {}; | |
| this._contextSize = 0; | |
| this._gpuActive = false; | |
| this._backendName = ""; | |
| this._supportsVision = false; | |
| this._supportsAudio = false; | |
| this._loadedModelUrl = null; | |
| this._loadedModelOptions = null; | |
| this._loadedMmProjUrl = null; | |
| this._forgetLoraAdapters(); | |
| this._loadedDraftModel = null; | |
| if (!this._disposed && this._lifecycleState === "open" && !this._workerProxy) { | |
| this._runtime = this._createRuntime(); | |
| operation?.runtimes?.add(this._runtime); | |
| } | |
| throw error; | |
| } finally { | |
| if (operation && this._activeOperation === operation && operation.state === "recovering") { | |
| operation.state = operation.cancelRequested ? "cancelling" : "running"; | |
| } | |
| } | |
| } | |
| _createWorkerProxy() { | |
| return new BridgeWorkerProxy({ | |
| moduleUrl: this._workerModuleUrl(), | |
| config: this._workerConfig() | |
| }); | |
| } | |
| _shouldUseWorker() { | |
| if (this._config?.disableWorker === true) { | |
| return false; | |
| } | |
| if (typeof Worker === "undefined" || typeof Blob === "undefined" || typeof URL === "undefined" || typeof URL.createObjectURL !== "function") { | |
| return false; | |
| } | |
| if (this._config?.coreModuleFactory != null) { | |
| return false; | |
| } | |
| return true; | |
| } | |
| _workerModuleUrl() { | |
| const candidate = this._config?.workerUrl; | |
| if (typeof candidate === "string" && candidate.trim().length > 0) { | |
| return candidate.trim(); | |
| } | |
| try { | |
| return new URL("./llama_webgpu_bridge_worker.js", import.meta.url).toString(); | |
| } catch (_) { | |
| return import.meta.url; | |
| } | |
| } | |
| _workerConfig() { | |
| const config = this._config || {}; | |
| return { | |
| workerRequestTimeoutMs: config.workerRequestTimeoutMs, | |
| workerInitTimeoutMs: config.workerInitTimeoutMs, | |
| workerModelLoadTimeoutMs: config.workerModelLoadTimeoutMs, | |
| workerMmprojLoadTimeoutMs: config.workerMmprojLoadTimeoutMs, | |
| workerCompletionTimeoutMs: config.workerCompletionTimeoutMs, | |
| workerTextToSpeechTimeoutMs: config.workerTextToSpeechTimeoutMs, | |
| coreInitTimeoutMs: config.coreInitTimeoutMs, | |
| wasmUrl: typeof config.wasmUrl === "string" ? config.wasmUrl : void 0, | |
| wasmUrlMem64: typeof config.wasmUrlMem64 === "string" ? config.wasmUrlMem64 : void 0, | |
| coreModuleUrl: typeof config.coreModuleUrl === "string" ? config.coreModuleUrl : void 0, | |
| coreModuleUrlMem64: typeof config.coreModuleUrlMem64 === "string" ? config.coreModuleUrlMem64 : void 0, | |
| preferMemory64: typeof config.preferMemory64 === "boolean" ? config.preferMemory64 : void 0, | |
| threads: Number(config.threads) > 0 ? Number(config.threads) : void 0, | |
| threadsBatch: Number(config.threadsBatch) > 0 ? Number(config.threadsBatch) : void 0, | |
| threadPoolSize: Number(config.threadPoolSize) > 0 ? Number(config.threadPoolSize) : void 0, | |
| nBatch: Number(config.nBatch) > 0 ? Number(config.nBatch) : void 0, | |
| nUbatch: Number(config.nUbatch) > 0 ? Number(config.nUbatch) : void 0, | |
| nGpuLayers: Number.isFinite(config.nGpuLayers) ? Number(config.nGpuLayers) : void 0, | |
| userAgent: typeof config.userAgent === "string" ? config.userAgent : void 0, | |
| cacheName: typeof config.cacheName === "string" ? config.cacheName : void 0, | |
| remoteFetchThresholdBytes: Number(config.remoteFetchThresholdBytes) > 0 ? Number(config.remoteFetchThresholdBytes) : void 0, | |
| remoteFetchChunkBytes: Number(config.remoteFetchChunkBytes) > 0 ? Number(config.remoteFetchChunkBytes) : void 0, | |
| mediaMaxImagePixels: Number(config.mediaMaxImagePixels) > 0 ? Number(config.mediaMaxImagePixels) : void 0, | |
| mediaMaxImageEdge: Number(config.mediaMaxImageEdge) > 0 ? Number(config.mediaMaxImageEdge) : void 0, | |
| disableImageDownscale: config.disableImageDownscale === true, | |
| allowAutoRemoteFetchBackend: config.allowAutoRemoteFetchBackend === true, | |
| logLevel: Number.isFinite(config.logLevel) ? Number(config.logLevel) : 2 | |
| }; | |
| } | |
| _applyShadowState(state) { | |
| if (!state || typeof state !== "object") { | |
| return; | |
| } | |
| if (state.metadata && typeof state.metadata === "object") { | |
| this._metadata = state.metadata; | |
| } | |
| if (Number.isFinite(state.contextSize)) { | |
| this._contextSize = Number(state.contextSize); | |
| } | |
| if (typeof state.gpuActive === "boolean") { | |
| this._gpuActive = state.gpuActive; | |
| } | |
| if (typeof state.backendName === "string" && state.backendName.length > 0) { | |
| this._backendName = state.backendName; | |
| } | |
| if (typeof state.supportsVision === "boolean") { | |
| this._supportsVision = state.supportsVision; | |
| } | |
| if (typeof state.supportsAudio === "boolean") { | |
| this._supportsAudio = state.supportsAudio; | |
| } | |
| } | |
| _shouldFallbackToMainThread(error) { | |
| const text = serializeWorkerError(error).toLowerCase(); | |
| if (text.includes("aborted(native code called abort())")) { | |
| return false; | |
| } | |
| if (text.includes("array buffer allocation failed")) { | |
| return false; | |
| } | |
| if (text.includes("bad_alloc")) { | |
| return false; | |
| } | |
| if (text.includes("out of memory")) { | |
| return false; | |
| } | |
| if (text.includes("memory access out of bounds")) { | |
| return false; | |
| } | |
| if (text.includes("bridge worker")) { | |
| return true; | |
| } | |
| if (text.includes("worker request failed")) { | |
| return true; | |
| } | |
| if (text.includes("worker request timeout")) { | |
| return true; | |
| } | |
| if (text.includes("worker init timeout")) { | |
| return true; | |
| } | |
| if (text.includes("timed out")) { | |
| return true; | |
| } | |
| if (text.includes("worker proxy is not available")) { | |
| return true; | |
| } | |
| if (text.includes("worker is not initialized")) { | |
| return true; | |
| } | |
| if (text.includes("failed to initialize bridge worker")) { | |
| return true; | |
| } | |
| if (text.includes("script error")) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| _resolvedBridgeLogLevel() { | |
| const configured = Number(this._config?.logLevel); | |
| if (Number.isFinite(configured)) { | |
| return Math.max(0, Math.min(4, Math.trunc(configured))); | |
| } | |
| const runtimeLevel = Number(this._runtime?._logLevel); | |
| if (Number.isFinite(runtimeLevel)) { | |
| return Math.max(0, Math.min(4, Math.trunc(runtimeLevel))); | |
| } | |
| return 2; | |
| } | |
| _shouldEmitBridgeLevel(level) { | |
| const configured = this._resolvedBridgeLogLevel(); | |
| const threshold = logThresholdForConfiguredLevel(configured); | |
| if (threshold > 3) { | |
| return false; | |
| } | |
| return logLevelForName(level) >= threshold; | |
| } | |
| _shouldSuppressBridgeWarn(message) { | |
| const text = String(message || "").trim(); | |
| if (text.length === 0) { | |
| return false; | |
| } | |
| const configuredWindow = Number(this._config?.warnDedupWindowMs); | |
| const dedupWindowMs = Number.isFinite(configuredWindow) && configuredWindow > 0 ? Math.max(500, Math.min(6e4, Math.trunc(configuredWindow))) : 5e3; | |
| const now = Date.now(); | |
| const last = Number(this._bridgeWarnRecent.get(text) || 0); | |
| this._bridgeWarnRecent.set(text, now); | |
| if (this._bridgeWarnRecent.size > 80) { | |
| const staleThreshold = now - dedupWindowMs * 2; | |
| for (const [key, atMs] of this._bridgeWarnRecent.entries()) { | |
| if (Number(atMs) < staleThreshold) { | |
| this._bridgeWarnRecent.delete(key); | |
| } | |
| } | |
| } | |
| return last > 0 && now - last < dedupWindowMs; | |
| } | |
| _emitBridgeWarn(message) { | |
| if (!this._shouldEmitBridgeLevel("warn")) { | |
| return; | |
| } | |
| if (this._shouldSuppressBridgeWarn(message)) { | |
| return; | |
| } | |
| if (this._runtime && typeof this._runtime._emitLogger === "function") { | |
| this._runtime._emitLogger("warn", message); | |
| return; | |
| } | |
| if (typeof console !== "undefined" && typeof console.warn === "function") { | |
| console.warn(message); | |
| } | |
| } | |
| _disableWorkerFallback(error) { | |
| const reason = serializeWorkerError(error); | |
| this._workerFallbackReason = reason; | |
| if (typeof globalThis !== "undefined") { | |
| globalThis.__llamadartBridgeWorkerFallbackReason = reason; | |
| } | |
| this._emitBridgeWarn( | |
| `llamadart: bridge worker unavailable, falling back to main thread (${reason})` | |
| ); | |
| if (this._workerProxy) { | |
| const workerProxy = this._workerProxy; | |
| this._advanceWorkerGeneration(); | |
| this._workerProxy = null; | |
| this._retireWorkerProxy(workerProxy); | |
| } | |
| this._multimodalWorkerCpuMode = false; | |
| this._workerModelMissing = false; | |
| const activeOperationMayFinish = this._activeOperation && this._activeOperation.kind !== "dispose" && this._activeOperation.state !== "cancelled" && this._activeOperation.state !== "disposed"; | |
| if ((this._disposed || this._lifecycleState === "disposing") && !activeOperationMayFinish) { | |
| return; | |
| } | |
| if (!this._runtime) { | |
| this._runtime = this._createRuntime(); | |
| this._activeOperation?.runtimes?.add(this._runtime); | |
| } | |
| const note = `worker_fallback:${reason}`; | |
| if (this._runtime && Array.isArray(this._runtime._runtimeNotes) && typeof reason === "string" && reason.length > 0 && !this._runtime._runtimeNotes.includes(note)) { | |
| this._runtime._runtimeNotes.push(note); | |
| } | |
| } | |
| async _callWorker(method, args, onEvent, transferList = []) { | |
| const proxy = this._workerProxy; | |
| if (!proxy) { | |
| throw new Error("Bridge worker proxy is not available"); | |
| } | |
| const operation = this._activeOperation; | |
| if (operation) { | |
| this._recordWorkerGeneration(operation, proxy); | |
| } | |
| const guardedEvent = typeof onEvent === "function" ? (event) => { | |
| if (!operation || this._isOperationCallbackCurrent(operation, proxy) || event.event === "usage" && proxy === this._workerProxy) { | |
| onEvent(event); | |
| } | |
| } : void 0; | |
| try { | |
| const response = await proxy.call( | |
| method, | |
| args, | |
| guardedEvent, | |
| transferList, | |
| operation ? { operationId: operation.id, workerGeneration: operation.workerGeneration } : void 0 | |
| ); | |
| if (response?.state) { | |
| this._acceptWorkerState(operation, proxy, response.state); | |
| } | |
| return response?.value; | |
| } catch (error) { | |
| if (error && typeof error === "object" && error.state) { | |
| this._acceptWorkerState(operation, proxy, error.state); | |
| } | |
| throw error; | |
| } | |
| } | |
| /** | |
| * A worker response may only reach facade state while its proxy is still the | |
| * operation's current owner. Staging it for an open recovery transaction is | |
| * the same commit deferred, so a retired generation must be rejected before | |
| * the transaction branch, not after it. | |
| */ | |
| _acceptWorkerState(operation, proxy, state) { | |
| if (operation && !this._isOperationCallbackCurrent(operation, proxy)) { | |
| return; | |
| } | |
| if (this._shadowStateTransactionDepth > 0) { | |
| this._deferredShadowState = state; | |
| if (operation) { | |
| operation.deferredShadowState = state; | |
| } | |
| return; | |
| } | |
| this._applyShadowStateFrom(proxy, state); | |
| } | |
| /** | |
| * The getters now answer from the facade snapshot, so a response from a proxy | |
| * that has since been replaced must not write into the current worker's state. | |
| */ | |
| _applyShadowStateFrom(proxy, state) { | |
| if (proxy !== this._workerProxy) { | |
| return; | |
| } | |
| this._applyShadowState(state); | |
| } | |
| async loadModelFromUrl(url, options = {}) { | |
| return this._runExclusive( | |
| () => this._loadModelFromUrlUnlocked(url, options), | |
| { | |
| signal: options?.signal, | |
| abortMessage: "Model load was cancelled.", | |
| kind: "load-model" | |
| } | |
| ); | |
| } | |
| async _loadModelFromUrlUnlocked(url, options = {}) { | |
| const attempt = { proxy: null, state: null, loaded: false }; | |
| try { | |
| return await this._loadModelFromUrlOnTarget(url, options, attempt); | |
| } catch (error) { | |
| this._syncLoadedModelAfterFailedLoad(attempt); | |
| throw error; | |
| } | |
| } | |
| /** | |
| * @param attempt Records the last worker load's proxy and reply for | |
| * `_syncLoadedModelAfterFailedLoad`. | |
| */ | |
| async _loadModelFromUrlOnTarget(url, options, attempt) { | |
| if (!this._workerProxy) { | |
| const runtimeOptions = { | |
| ...options, | |
| signal: this._operationSignal() || options.signal || null | |
| }; | |
| const result = await this._runtime.loadModelFromUrl(url, runtimeOptions); | |
| this._rememberLoadedModel(url, options); | |
| return result; | |
| } | |
| const invokeWorkerLoad = async () => { | |
| const workerOptions = { ...options }; | |
| delete workerOptions.progressCallback; | |
| delete workerOptions.signal; | |
| const proxy = this._workerProxy; | |
| attempt.proxy = proxy; | |
| attempt.state = null; | |
| attempt.loaded = false; | |
| let result; | |
| try { | |
| result = await this._callWorker( | |
| "loadModelFromUrl", | |
| [url, workerOptions], | |
| (event) => { | |
| if (event.event !== "progress") { | |
| return; | |
| } | |
| if (typeof options.progressCallback !== "function") { | |
| return; | |
| } | |
| options.progressCallback(event.payload || {}); | |
| } | |
| ); | |
| } catch (error) { | |
| attempt.state = error && typeof error === "object" ? error.state || null : null; | |
| throw error; | |
| } | |
| attempt.loaded = true; | |
| this._throwIfCallerCancelled("Model load was cancelled."); | |
| this._rememberLoadedModel(url, workerOptions); | |
| return result; | |
| }; | |
| try { | |
| return await invokeWorkerLoad(); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Model load was cancelled."); | |
| if (this._isRecoverableWorkerFsError(error) && !this._isWorkerRequestTimeoutError(error)) { | |
| this._emitBridgeWarn( | |
| `llamadart: worker model-load FS error detected; restarting worker (${serializeWorkerError(error)}).` | |
| ); | |
| try { | |
| await this._replaceWorkerProxyForMultimodalCpuMode(); | |
| this._forgetLoadedModel(); | |
| this._applyShadowState(emptyBridgeState()); | |
| return await invokeWorkerLoad(); | |
| } catch (retryError) { | |
| this._emitBridgeWarn( | |
| `llamadart: worker model-load retry failed (${serializeWorkerError(retryError)}).` | |
| ); | |
| error = retryError; | |
| } | |
| } | |
| if (!this._shouldFallbackToMainThread(error)) { | |
| throw error; | |
| } | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| this._throwIfDisposed(); | |
| const result = await this._runtime.loadModelFromUrl(url, { | |
| ...options, | |
| signal: this._operationSignal() || options.signal || null | |
| }); | |
| this._rememberLoadedModel(url, options); | |
| return result; | |
| } | |
| } | |
| async prefetchModelToCache(url, options = {}) { | |
| this._throwIfDisposed(); | |
| if (!this._runtime) { | |
| this._runtime = this._createRuntime(); | |
| } | |
| try { | |
| return await this._runtime.prefetchModelToCache(url, options); | |
| } finally { | |
| this._refreshCacheMetadataSnapshot(); | |
| } | |
| } | |
| async evictModelFromCache(url, options = {}) { | |
| this._throwIfDisposed(); | |
| if (!this._runtime) { | |
| this._runtime = this._createRuntime(); | |
| } | |
| try { | |
| return await this._runtime.evictModelFromCache(url, options); | |
| } finally { | |
| this._refreshCacheMetadataSnapshot(); | |
| } | |
| } | |
| async createCompletion(prompt, options = {}) { | |
| resolveCompletionSamplingOptions(options); | |
| const onUsage = options?.onUsage; | |
| if (typeof onUsage !== "function") { | |
| return this._runExclusive( | |
| () => this._createCompletionUnlocked(prompt, options), | |
| { | |
| signal: options?.signal, | |
| abortMessage: "Generation was cancelled.", | |
| kind: "generation" | |
| } | |
| ); | |
| } | |
| let usage = null; | |
| let text; | |
| try { | |
| text = await this._runExclusive( | |
| () => this._createCompletionUnlocked(prompt, { | |
| ...options, | |
| onUsage: (value) => { | |
| usage = value; | |
| } | |
| }), | |
| { | |
| signal: options.signal, | |
| abortMessage: "Generation was cancelled.", | |
| kind: "generation" | |
| } | |
| ); | |
| } catch (error) { | |
| if (usage != null && error?.name === "AbortError") { | |
| onUsage(usage); | |
| } | |
| throw error; | |
| } | |
| if (usage != null) { | |
| onUsage(usage); | |
| } | |
| return text; | |
| } | |
| async getCompletionCapabilities() { | |
| return this._runExclusive( | |
| () => this._getCompletionCapabilitiesUnlocked(), | |
| { kind: "completion-capabilities" } | |
| ); | |
| } | |
| async _getCompletionCapabilitiesUnlocked() { | |
| if (!this._workerProxy) { | |
| return this._runtime?.getCompletionCapabilities() ?? { ...NO_COMPLETION_CAPABILITIES }; | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| return await this._callWorker("getCompletionCapabilities", []); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Completion capability probe was cancelled."); | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback({}, error); | |
| return this._runtime.getCompletionCapabilities(); | |
| } | |
| } | |
| async _createCompletionUnlocked(prompt, options = {}) { | |
| const isWarmup = options?.warmup === true; | |
| const hasRetriedEmptyMultimodal = options?.__llamadartEmptyRetryAttempted === true; | |
| const workerAllowed = this._config?.disableWorker !== true; | |
| if (this._hasMediaParts(options) && workerAllowed) { | |
| const hasWorkerFallback = typeof this._workerFallbackReason === "string" && this._workerFallbackReason.length > 0; | |
| if (hasWorkerFallback && !this._workerProxy && !this._isCpuModelMode()) { | |
| await this._ensureRuntimeReadyAfterWorkerFallback(options, null); | |
| return this._runtime.createCompletion(prompt, options); | |
| } | |
| try { | |
| if (!this._workerProxy) { | |
| await this._replaceWorkerProxyForMultimodalCpuMode(); | |
| } | |
| await this._ensureWorkerMultimodalCpuMode(); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Generation was cancelled."); | |
| const reason = serializeWorkerError(error); | |
| if (isWarmup) { | |
| this._emitBridgeWarn( | |
| `llamadart: multimodal warmup skipped after worker setup issue (${reason}).` | |
| ); | |
| return ""; | |
| } | |
| this._emitBridgeWarn( | |
| `llamadart: unable to prepare multimodal worker CPU mode (${reason}).` | |
| ); | |
| if (this._isCpuModelMode()) { | |
| throw new Error( | |
| `CPU multimodal worker setup failed (${reason}). Reload model and retry with a smaller image.` | |
| ); | |
| } | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback(options, error); | |
| return this._runtime.createCompletion(prompt, options); | |
| } | |
| } | |
| if (!this._workerProxy) { | |
| return this._runtime.createCompletion(prompt, options); | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| const workerOptions = { ...options }; | |
| delete workerOptions.onToken; | |
| delete workerOptions.onUsage; | |
| delete workerOptions.signal; | |
| delete workerOptions.__llamadartEmptyRetryAttempted; | |
| const speculative = options.speculativeDecoding; | |
| if (speculative != null && typeof speculative === "object" && !Array.isArray(speculative)) { | |
| const resolveCache = (source) => typeof source === "string" && source.length > 0 ? normalizeAbsoluteUrl(source) : source; | |
| workerOptions.speculativeDecoding = { | |
| ...speculative, | |
| ngramCacheStatic: resolveCache(speculative.ngramCacheStatic), | |
| ngramCacheDynamic: resolveCache(speculative.ngramCacheDynamic) | |
| }; | |
| } | |
| const stallTimeoutMs = this._workerCompletionStallTimeoutMs(options); | |
| let timeoutHandle = null; | |
| let rejectOnStall = null; | |
| const clearStallTimer = () => { | |
| if (timeoutHandle != null) { | |
| globalThis.clearTimeout(timeoutHandle); | |
| timeoutHandle = null; | |
| } | |
| }; | |
| const armStallTimer = () => { | |
| clearStallTimer(); | |
| if (!Number.isFinite(stallTimeoutMs) || stallTimeoutMs <= 0) { | |
| return; | |
| } | |
| timeoutHandle = globalThis.setTimeout(() => { | |
| const timeoutError = new Error( | |
| `Bridge worker completion stalled for ${stallTimeoutMs}ms.` | |
| ); | |
| timeoutError.llamadartWorkerTimeout = true; | |
| this._cancelOperation(this._activeOperation, "worker-timeout"); | |
| rejectOnStall?.(timeoutError); | |
| }, stallTimeoutMs); | |
| }; | |
| const stallPromise = new Promise((_, reject) => { | |
| rejectOnStall = reject; | |
| }); | |
| armStallTimer(); | |
| let sawWorkerTokenEvent = false; | |
| try { | |
| const workerResult = await Promise.race([ | |
| this._callWorker( | |
| "createCompletion", | |
| [prompt, workerOptions], | |
| (event) => { | |
| if (event.event === "usage") { | |
| if (typeof options.onUsage === "function") { | |
| options.onUsage(event.payload); | |
| } | |
| return; | |
| } | |
| if (event.event !== "token") { | |
| return; | |
| } | |
| armStallTimer(); | |
| sawWorkerTokenEvent = true; | |
| if (typeof options.onToken !== "function") { | |
| return; | |
| } | |
| const payload = event.payload || {}; | |
| const piece = typeof payload.pieceText === "string" ? payload.pieceText : Uint8Array.from(Array.isArray(payload.piece) ? payload.piece : []); | |
| options.onToken(piece, String(payload.currentText || "")); | |
| } | |
| ), | |
| stallPromise | |
| ]); | |
| if (this._hasMediaParts(options) && !isWarmup && !sawWorkerTokenEvent && String(workerResult || "").trim().length == 0) { | |
| this._emitBridgeWarn( | |
| "llamadart: multimodal worker produced empty response without token events." | |
| ); | |
| if (!hasRetriedEmptyMultimodal) { | |
| this._emitBridgeWarn( | |
| "llamadart: retrying multimodal worker once after empty response." | |
| ); | |
| try { | |
| await this._replaceWorkerProxyForMultimodalCpuMode(); | |
| await this._ensureWorkerMultimodalCpuMode(); | |
| } catch (retrySetupError) { | |
| this._emitBridgeWarn( | |
| `llamadart: multimodal empty-response retry setup failed (${serializeWorkerError(retrySetupError)}).` | |
| ); | |
| } | |
| return this._createCompletionUnlocked(prompt, { | |
| ...options, | |
| __llamadartEmptyRetryAttempted: true | |
| }); | |
| } | |
| } | |
| this._throwIfCallerCancelled("Generation was cancelled."); | |
| return workerResult; | |
| } finally { | |
| clearStallTimer(); | |
| } | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Generation was cancelled."); | |
| if (serializeWorkerError(error).includes(INVALID_GRAMMAR_ERROR_TEXT)) { | |
| throw error; | |
| } | |
| if (this._hasMediaParts(options)) { | |
| const reason = serializeWorkerError(error); | |
| if (isWarmup) { | |
| this._emitBridgeWarn( | |
| `llamadart: multimodal warmup skipped after worker request issue (${reason}).` | |
| ); | |
| return ""; | |
| } | |
| if (this._isCpuModelMode()) { | |
| this._emitBridgeWarn( | |
| `llamadart: CPU multimodal worker request failed (${reason}); skipping main-thread fallback.` | |
| ); | |
| throw new Error( | |
| `CPU multimodal request failed (${reason}). Reload model and retry with a smaller image.` | |
| ); | |
| } | |
| this._emitBridgeWarn( | |
| `llamadart: multimodal worker request failed (${reason}); falling back to main-thread runtime.` | |
| ); | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback(options, error); | |
| return this._runtime.createCompletion(prompt, options); | |
| } | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback(options, error); | |
| return this._runtime.createCompletion(prompt, options); | |
| } | |
| } | |
| async loadMultimodalProjector(url) { | |
| return this._runExclusive( | |
| () => this._loadMultimodalProjectorUnlocked(url), | |
| { kind: "projector-load" } | |
| ); | |
| } | |
| async _loadMultimodalProjectorUnlocked(url) { | |
| const startedWithWorker = this._workerProxy != null; | |
| const invokeRuntimeLoad = async () => { | |
| if (!this._runtime) { | |
| this._runtime = this._createRuntime(); | |
| } | |
| await this._ensureRuntimeReadyAfterWorkerFallback({}, null); | |
| const result = await this._runtime.loadMultimodalProjector(url); | |
| this._rememberLoadedMmProj(url); | |
| this._supportsVision = this._runtime.supportsVision(); | |
| this._supportsAudio = this._runtime.supportsAudio(); | |
| return result; | |
| }; | |
| try { | |
| if (!this._workerProxy) { | |
| return await invokeRuntimeLoad(); | |
| } | |
| await this._ensureWorkerMultimodalCpuMode(); | |
| const result = await this._callWorker("loadMultimodalProjector", [url]); | |
| this._rememberLoadedMmProj(url); | |
| return result; | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Projector operation was cancelled."); | |
| if (!startedWithWorker) { | |
| throw error; | |
| } | |
| const reason = serializeWorkerError(error); | |
| this._emitBridgeWarn( | |
| `llamadart: multimodal worker setup failed (${reason}).` | |
| ); | |
| if (this._isCpuModelMode()) { | |
| try { | |
| await this._replaceWorkerProxyForMultimodalCpuMode(); | |
| await this._ensureWorkerMultimodalCpuMode(); | |
| const retryResult = await this._callWorker("loadMultimodalProjector", [url]); | |
| this._rememberLoadedMmProj(url); | |
| this._emitBridgeWarn( | |
| "llamadart: CPU multimodal worker setup recovered after worker restart." | |
| ); | |
| return retryResult; | |
| } catch (retryError) { | |
| const retryReason = serializeWorkerError(retryError); | |
| throw new Error( | |
| `CPU multimodal projector setup failed (${retryReason}). Reload model and retry with a smaller image.` | |
| ); | |
| } | |
| } | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| return invokeRuntimeLoad(); | |
| } | |
| } | |
| async unloadMultimodalProjector() { | |
| return this._runExclusive( | |
| () => this._unloadMultimodalProjectorUnlocked(), | |
| { kind: "projector-unload" } | |
| ); | |
| } | |
| async _unloadMultimodalProjectorUnlocked() { | |
| if (!this._workerProxy) { | |
| const result = await this._runtime.unloadMultimodalProjector(); | |
| this._loadedMmProjUrl = null; | |
| this._supportsVision = this._runtime.supportsVision(); | |
| this._supportsAudio = this._runtime.supportsAudio(); | |
| return result; | |
| } | |
| try { | |
| const result = await this._callWorker("unloadMultimodalProjector", []); | |
| this._loadedMmProjUrl = null; | |
| return result; | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Projector operation was cancelled."); | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| const result = await this._runtime.unloadMultimodalProjector(); | |
| this._loadedMmProjUrl = null; | |
| this._supportsVision = this._runtime.supportsVision(); | |
| this._supportsAudio = this._runtime.supportsAudio(); | |
| return result; | |
| } | |
| } | |
| // Loads the remembered draft model into the direct runtime when that | |
| // runtime does not hold it. | |
| async _ensureRuntimeDraftModel() { | |
| const draft = this._loadedDraftModel; | |
| const runtime = this._runtime; | |
| if (!draft || !runtime || runtime._draftModel?.url === draft.url) { | |
| return; | |
| } | |
| this._throwIfDisposed(); | |
| try { | |
| await runtime.loadDraftModel(draft.url, { | |
| ...draft.options, | |
| signal: this._operationSignal() || void 0 | |
| }); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Draft model reload was cancelled."); | |
| this._forgetDraftModelAfterReloadFailure(error); | |
| } | |
| this._throwIfDisposed(); | |
| } | |
| // A reloaded target starts without its draft model. A failed draft reload | |
| // leaves none, so draft-* strategies then reject until loadDraftModel runs. | |
| _forgetDraftModelAfterReloadFailure(error) { | |
| this._loadedDraftModel = null; | |
| this._emitBridgeWarn( | |
| `llamadart: draft model reload failed after the model was reloaded (${serializeWorkerError(error)}); call loadDraftModel again.` | |
| ); | |
| } | |
| async loadDraftModel(url, options = {}) { | |
| return this._runExclusive( | |
| () => this._loadDraftModelUnlocked(url, options), | |
| { | |
| signal: options?.signal, | |
| abortMessage: "Draft model load was cancelled.", | |
| kind: "draft-model-load" | |
| } | |
| ); | |
| } | |
| async _loadDraftModelUnlocked(url, options = {}) { | |
| const absoluteUrl = typeof url === "string" && url.length > 0 ? normalizeAbsoluteUrl(url) : url; | |
| const remembered = { | |
| url: absoluteUrl, | |
| options: typeof options?.useCache === "boolean" ? { useCache: options.useCache } : {} | |
| }; | |
| const loadInRuntime = async () => { | |
| this._loadedDraftModel = null; | |
| const info = await this._runtime.loadDraftModel(absoluteUrl, { | |
| ...options, | |
| signal: this._operationSignal() || options.signal || void 0 | |
| }); | |
| this._loadedDraftModel = remembered; | |
| return info; | |
| }; | |
| if (!this._workerProxy) { | |
| return loadInRuntime(); | |
| } | |
| const workerOptions = { ...options }; | |
| delete workerOptions.progressCallback; | |
| delete workerOptions.signal; | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| this._loadedDraftModel = null; | |
| const info = await this._callWorker( | |
| "loadDraftModel", | |
| [absoluteUrl, workerOptions], | |
| (event) => { | |
| if (event.event === "progress" && typeof options.progressCallback === "function") { | |
| options.progressCallback(event.payload || {}); | |
| } | |
| } | |
| ); | |
| this._throwIfCallerCancelled("Draft model load was cancelled."); | |
| this._loadedDraftModel = remembered; | |
| return info; | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Draft model load was cancelled."); | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback({}, error); | |
| return loadInRuntime(); | |
| } | |
| } | |
| async unloadDraftModel() { | |
| return this._runExclusive( | |
| () => this._unloadDraftModelUnlocked(), | |
| { kind: "draft-model-unload" } | |
| ); | |
| } | |
| async _unloadDraftModelUnlocked() { | |
| if (!this._workerProxy) { | |
| await this._runtime?.unloadDraftModel(); | |
| this._loadedDraftModel = null; | |
| return; | |
| } | |
| try { | |
| await this._callWorker("unloadDraftModel", []); | |
| this._loadedDraftModel = null; | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Draft model unload was cancelled."); | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| this._loadedDraftModel = null; | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._runtime?.unloadDraftModel(); | |
| } | |
| } | |
| supportsVision() { | |
| return this._supportsVision; | |
| } | |
| supportsAudio() { | |
| return this._supportsAudio; | |
| } | |
| async getTextToSpeechCapabilities() { | |
| return this._runExclusive( | |
| () => this._getTextToSpeechCapabilitiesUnlocked(), | |
| { kind: "tts-capabilities" } | |
| ); | |
| } | |
| async _getTextToSpeechCapabilitiesUnlocked() { | |
| if (!this._workerProxy) { | |
| return this._runtime.getTextToSpeechCapabilities(); | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| return await this._callWorker("getTextToSpeechCapabilities", []); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Text-to-speech capability probe was cancelled."); | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback({}, error); | |
| return this._runtime.getTextToSpeechCapabilities(); | |
| } | |
| } | |
| async synthesizeSpeech(options = {}) { | |
| return this._runExclusive( | |
| () => this._synthesizeSpeechUnlocked(options), | |
| { | |
| signal: options?.signal, | |
| abortMessage: "Text-to-speech synthesis was cancelled.", | |
| kind: "text-to-speech" | |
| } | |
| ); | |
| } | |
| async _synthesizeSpeechUnlocked(options = {}) { | |
| if (options.signal?.aborted) { | |
| throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); | |
| } | |
| if (!this._workerProxy) { | |
| return this._runtime.synthesizeSpeech(options); | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| const workerOptions = { ...options }; | |
| delete workerOptions.onProgress; | |
| delete workerOptions.signal; | |
| const transferList = []; | |
| const speakerBytes = toUint8Array(options.speakerAudio); | |
| if (speakerBytes && speakerBytes.length > 0) { | |
| const copy = new Uint8Array(speakerBytes); | |
| workerOptions.speakerAudio = copy; | |
| transferList.push(copy.buffer); | |
| } | |
| const result = await this._callWorker( | |
| "synthesizeSpeech", | |
| [workerOptions], | |
| (event) => { | |
| if (event.event === "progress") { | |
| options.onProgress?.(event.payload || {}); | |
| } | |
| }, | |
| transferList | |
| ); | |
| this._throwIfCallerCancelled("Text-to-speech synthesis was cancelled."); | |
| return result; | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Text-to-speech synthesis was cancelled."); | |
| if (error?.name === "AbortError" || options.signal?.aborted || serializeWorkerError(error).toLowerCase().includes("cancel")) { | |
| throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); | |
| } | |
| const canRetryOnCpu = !this._isCpuModelMode() && (this._isDispatchWorkgroupLimitError(error) || this._isWorkerRequestTimeoutError(error)); | |
| if (canRetryOnCpu) { | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback( | |
| { | |
| _llamadartForceRuntimeReload: true, | |
| _llamadartTextToSpeech: true | |
| }, | |
| error | |
| ); | |
| if (options.signal?.aborted) { | |
| throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError"); | |
| } | |
| return this._runtime.synthesizeSpeech(options); | |
| } | |
| if (this._isWorkerRequestTimeoutError(error)) { | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| } | |
| throw error; | |
| } | |
| } | |
| _decisionHeadMap() { | |
| if (!(this._decisionHeads instanceof Map)) { | |
| this._decisionHeads = /* @__PURE__ */ new Map(); | |
| this._nextDecisionHandle = 1; | |
| } | |
| return this._decisionHeads; | |
| } | |
| _registerDecisionHead(owner, info) { | |
| const heads = this._decisionHeadMap(); | |
| const handle = this._nextDecisionHandle++; | |
| heads.set(handle, { owner, handle: Number(info?.handle) }); | |
| return { ...info, handle }; | |
| } | |
| _decisionOwner() { | |
| return this._workerProxy || this._runtime || null; | |
| } | |
| _staleDecisionHeadError(handle) { | |
| return new Error( | |
| `Decision head ${handle} is not loaded; it was freed, its model was unloaded, or the bridge runtime restarted. Load the decision head again.` | |
| ); | |
| } | |
| _resolveDecisionHead(handle) { | |
| decisionHandleFrom(handle); | |
| const heads = this._decisionHeadMap(); | |
| const entry = heads.get(handle); | |
| if (!entry) { | |
| return null; | |
| } | |
| if (entry.owner !== this._decisionOwner()) { | |
| heads.delete(handle); | |
| return null; | |
| } | |
| return entry; | |
| } | |
| _requireDecisionRuntime() { | |
| if (!this._runtime) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| return this._runtime; | |
| } | |
| /** | |
| * Moves a failed worker's work to the main-thread runtime. Decision heads | |
| * the worker held are gone with it; the model, its projector and the applied | |
| * LoRA adapters are restored so later calls keep working. | |
| */ | |
| async _recoverWorkerOnMainThread(error) { | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback({}, error); | |
| } | |
| async getDecisionCapabilities() { | |
| return this._runExclusive( | |
| () => this._getDecisionCapabilitiesUnlocked(), | |
| { kind: "decision-capabilities" } | |
| ); | |
| } | |
| async _getDecisionCapabilitiesUnlocked() { | |
| if (!this._workerProxy) { | |
| return this._runtime ? this._runtime.getDecisionCapabilities() : { apiVersion: DECISION_API_VERSION, supported: false, reason: "WebGPU core is not initialized" }; | |
| } | |
| try { | |
| return await this._callWorker("getDecisionCapabilities", []); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Decision capability probe was cancelled."); | |
| if (!this._shouldFallbackToMainThread(error)) { | |
| throw error; | |
| } | |
| await this._recoverWorkerOnMainThread(error); | |
| return this._requireDecisionRuntime().getDecisionCapabilities(); | |
| } | |
| } | |
| async loadDecisionHead(source, options = {}) { | |
| return this._runExclusive( | |
| () => this._loadDecisionHeadUnlocked(source, options), | |
| { kind: "decision-head-load" } | |
| ); | |
| } | |
| async _loadDecisionHeadUnlocked(source, options = {}) { | |
| const loadInRuntime = async () => { | |
| const runtime = this._requireDecisionRuntime(); | |
| return this._registerDecisionHead( | |
| runtime, | |
| await runtime.loadDecisionHead(source, options) | |
| ); | |
| }; | |
| if (!this._workerProxy) { | |
| return loadInRuntime(); | |
| } | |
| const workerOptions = { configJson: options?.configJson ?? null }; | |
| let workerSource = source; | |
| const transferList = []; | |
| if (typeof source === "string") { | |
| workerSource = source.length > 0 ? normalizeAbsoluteUrl(source) : source; | |
| } else { | |
| const bytes = bufferSourceBytes(source); | |
| if (bytes) { | |
| const copy = new Uint8Array(bytes); | |
| workerSource = copy; | |
| transferList.push(copy.buffer); | |
| } | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| const proxy = this._workerProxy; | |
| const info = await this._callWorker( | |
| "loadDecisionHead", | |
| [workerSource, workerOptions], | |
| (event) => { | |
| if (event.event === "progress") { | |
| options?.onProgress?.(event.payload || {}); | |
| } | |
| }, | |
| transferList | |
| ); | |
| return this._registerDecisionHead(proxy, info); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Decision head load was cancelled."); | |
| if (!this._shouldFallbackToMainThread(error)) { | |
| throw error; | |
| } | |
| await this._recoverWorkerOnMainThread(error); | |
| return loadInRuntime(); | |
| } | |
| } | |
| async runDecision(handle, sequences) { | |
| return this._runExclusive( | |
| () => this._runDecisionUnlocked(handle, sequences), | |
| { kind: "decision-run" } | |
| ); | |
| } | |
| async _runDecisionUnlocked(handle, sequences) { | |
| const entry = this._resolveDecisionHead(handle); | |
| if (!entry) { | |
| throw this._staleDecisionHeadError(handle); | |
| } | |
| const normalized = normalizeDecisionSequences(sequences); | |
| if (!this._workerProxy) { | |
| return this._requireDecisionRuntime().runDecision(entry.handle, normalized); | |
| } | |
| try { | |
| return await this._callWorker("runDecision", [entry.handle, normalized]); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Decision run was cancelled."); | |
| if (!this._shouldFallbackToMainThread(error)) { | |
| throw error; | |
| } | |
| this._decisionHeadMap().delete(handle); | |
| await this._recoverWorkerOnMainThread(error); | |
| throw new Error( | |
| `Decision head ${handle} was lost when the bridge worker failed (${serializeWorkerError(error)}). Load the decision head again.` | |
| ); | |
| } | |
| } | |
| async freeDecisionHead(handle) { | |
| return this._runExclusive( | |
| () => this._freeDecisionHeadUnlocked(handle), | |
| { kind: "decision-head-free" } | |
| ); | |
| } | |
| async _freeDecisionHeadUnlocked(handle) { | |
| const entry = this._resolveDecisionHead(handle); | |
| if (!entry) { | |
| return; | |
| } | |
| this._decisionHeadMap().delete(handle); | |
| if (!this._workerProxy) { | |
| await this._requireDecisionRuntime().freeDecisionHead(entry.handle); | |
| return; | |
| } | |
| try { | |
| await this._callWorker("freeDecisionHead", [entry.handle]); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Decision head free was cancelled."); | |
| if (!this._shouldFallbackToMainThread(error)) { | |
| throw error; | |
| } | |
| await this._recoverWorkerOnMainThread(error); | |
| } | |
| } | |
| // Test doubles skip the constructor, so the LoRA state starts on first use. | |
| _loraAdapterMap() { | |
| if (!(this._loraAdapters instanceof Map)) { | |
| this._loraAdapters = /* @__PURE__ */ new Map(); | |
| this._activeLoraScales = /* @__PURE__ */ new Map(); | |
| this._nextLoraHandle = 1; | |
| } | |
| return this._loraAdapters; | |
| } | |
| _activeLoraScaleMap() { | |
| this._loraAdapterMap(); | |
| return this._activeLoraScales; | |
| } | |
| _forgetLoraAdapters() { | |
| this._loraAdapterMap().clear(); | |
| this._activeLoraScaleMap().clear(); | |
| } | |
| /** | |
| * Marks the adapters `owner` holds as lost before it loads the model again, | |
| * which frees them. An adapter with a retained source reloads on the next | |
| * restore or set; one without is forgotten. | |
| */ | |
| _dropLoraAdaptersOf(owner) { | |
| for (const [handle, entry] of this._loraAdapterMap()) { | |
| if (owner === null || entry.owner !== owner) { | |
| continue; | |
| } | |
| if (entry.source === null) { | |
| this._loraAdapterMap().delete(handle); | |
| this._activeLoraScaleMap().delete(handle); | |
| } else { | |
| entry.owner = null; | |
| } | |
| } | |
| } | |
| _loraOwner() { | |
| return this._workerProxy || this._runtime || null; | |
| } | |
| _requireLoraRuntime() { | |
| if (!this._runtime) { | |
| throw new Error("No model loaded. Call loadModelFromUrl first."); | |
| } | |
| return this._runtime; | |
| } | |
| _loraAdapterEntry(handle) { | |
| const entry = this._loraAdapterMap().get(handle); | |
| if (!entry) { | |
| throw staleLoraAdapterError(handle); | |
| } | |
| return entry; | |
| } | |
| // Loads an adapter into the current owner: the worker, or the direct runtime. | |
| async _loadLoraAdapterIntoOwner(source, options) { | |
| if (!this._workerProxy) { | |
| const runtime = this._requireLoraRuntime(); | |
| const info2 = await runtime.loadLoraAdapter(source, { | |
| ...options, | |
| signal: this._operationSignal() || options.signal || null | |
| }); | |
| return { owner: runtime, handle: info2.handle }; | |
| } | |
| const workerOptions = {}; | |
| if (options.useCache !== void 0) { | |
| workerOptions.useCache = options.useCache; | |
| } | |
| if (options.cacheName !== void 0) { | |
| workerOptions.cacheName = options.cacheName; | |
| } | |
| let workerSource = source; | |
| const transferList = []; | |
| if (typeof source !== "string") { | |
| workerSource = new Uint8Array(source); | |
| transferList.push(workerSource.buffer); | |
| } | |
| const proxy = this._workerProxy; | |
| const info = await this._callWorker( | |
| "loadLoraAdapter", | |
| [workerSource, workerOptions], | |
| (event) => { | |
| if (event.event === "progress") { | |
| options.progressCallback?.(event.payload || {}); | |
| } | |
| }, | |
| transferList | |
| ); | |
| return { owner: proxy, handle: Number(info?.handle) }; | |
| } | |
| // Reloads an adapter its owner lost, into the current owner. | |
| async _ensureLoraAdapterOnOwner(handle, entry) { | |
| if (entry.owner === this._loraOwner()) { | |
| return; | |
| } | |
| if (entry.source === null) { | |
| this._loraAdapterMap().delete(handle); | |
| this._activeLoraScaleMap().delete(handle); | |
| throw staleLoraAdapterError(handle); | |
| } | |
| const loaded = await this._loadLoraAdapterIntoOwner(entry.source, entry.cacheOptions); | |
| entry.owner = loaded.owner; | |
| entry.handle = loaded.handle; | |
| } | |
| async _callLoraOwner(method, args) { | |
| if (this._workerProxy) { | |
| await this._callWorker(method, args); | |
| return; | |
| } | |
| const runtime = this._requireLoraRuntime(); | |
| if (method === "setLoraAdapter") { | |
| await runtime.setLoraAdapter(args[0], args[1]); | |
| } else if (method === "removeLoraAdapter") { | |
| await runtime.removeLoraAdapter(args[0]); | |
| } else { | |
| await runtime.clearLoraAdapters(); | |
| } | |
| } | |
| /** | |
| * Makes the current owner apply exactly the facade's active adapters after a | |
| * worker restart or a fallback to the main thread. The owner's own set is | |
| * cleared first, since a runtime that held the model earlier may still apply | |
| * adapters from then. Inactive adapters reload when next set. | |
| */ | |
| async _restoreLoraAdapters() { | |
| const owner = this._loraOwner(); | |
| if (![...this._loraAdapterMap().values()].some((entry) => entry.owner !== owner)) { | |
| return; | |
| } | |
| await this._callLoraOwner("clearLoraAdapters", []); | |
| for (const [handle, scale] of [...this._activeLoraScaleMap()]) { | |
| const entry = this._loraAdapterEntry(handle); | |
| await this._ensureLoraAdapterOnOwner(handle, entry); | |
| await this._callLoraOwner("setLoraAdapter", [entry.handle, scale]); | |
| } | |
| } | |
| // Runs a LoRA call on the current owner, moving to the main-thread runtime | |
| // and running it there once if the worker fails. | |
| async _runLoraCall(call, cancelMessage) { | |
| if (!this._workerProxy) { | |
| return call(); | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| return await call(); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, cancelMessage); | |
| if (!this._shouldFallbackToMainThread(error)) { | |
| throw error; | |
| } | |
| await this._recoverWorkerOnMainThread(error); | |
| return call(); | |
| } | |
| } | |
| async getLoraAdapterCapabilities() { | |
| return this._runExclusive( | |
| () => this._getLoraAdapterCapabilitiesUnlocked(), | |
| { kind: "lora-capabilities" } | |
| ); | |
| } | |
| async _getLoraAdapterCapabilitiesUnlocked() { | |
| if (!this._workerProxy && !this._runtime) { | |
| return { apiVersion: LORA_API_VERSION, supported: false, reason: "WebGPU core is not initialized" }; | |
| } | |
| return this._runLoraCall( | |
| () => this._workerProxy ? this._callWorker("getLoraAdapterCapabilities", []) : Promise.resolve(this._requireLoraRuntime().getLoraAdapterCapabilities()), | |
| "LoRA capability probe was cancelled." | |
| ); | |
| } | |
| async loadLoraAdapter(source, options = {}) { | |
| return this._runExclusive( | |
| () => this._loadLoraAdapterUnlocked(source, options || {}), | |
| { | |
| signal: options?.signal, | |
| abortMessage: LORA_LOAD_ABORT_MESSAGE, | |
| kind: "lora-adapter-load" | |
| } | |
| ); | |
| } | |
| async _loadLoraAdapterUnlocked(source, options) { | |
| let normalized; | |
| if (typeof source === "string") { | |
| if (source.length === 0) { | |
| throw new Error("LoRA adapter URL is empty."); | |
| } | |
| normalized = normalizeAbsoluteUrl(source); | |
| } else { | |
| const bytes = bufferSourceBytes(source); | |
| if (!bytes) { | |
| throw new TypeError("LoRA adapter source must be a URL string, an ArrayBuffer or a typed array."); | |
| } | |
| if (bytes.byteLength === 0) { | |
| throw new Error("LoRA adapter bytes are empty."); | |
| } | |
| normalized = bytes; | |
| } | |
| let retained = null; | |
| if (this._config?.disableWorker !== true) { | |
| retained = typeof normalized === "string" ? normalized : new Uint8Array(normalized); | |
| } | |
| const loaded = await this._runLoraCall( | |
| () => this._loadLoraAdapterIntoOwner(retained ?? normalized, options), | |
| LORA_LOAD_ABORT_MESSAGE | |
| ); | |
| this._loraAdapterMap(); | |
| const handle = this._nextLoraHandle++; | |
| this._loraAdapterMap().set(handle, { | |
| owner: loaded.owner, | |
| handle: loaded.handle, | |
| source: retained, | |
| cacheOptions: { useCache: options.useCache, cacheName: options.cacheName } | |
| }); | |
| return { handle }; | |
| } | |
| async setLoraAdapter(handle, scale = 1) { | |
| const facadeHandle = loraHandleFrom(handle); | |
| const checkedScale = loraScaleFrom(scale); | |
| return this._runExclusive( | |
| () => this._setLoraAdapterUnlocked(facadeHandle, checkedScale), | |
| { kind: "lora-adapter-set" } | |
| ); | |
| } | |
| async _setLoraAdapterUnlocked(handle, scale) { | |
| const entry = this._loraAdapterEntry(handle); | |
| await this._runLoraCall(async () => { | |
| await this._ensureLoraAdapterOnOwner(handle, entry); | |
| await this._callLoraOwner("setLoraAdapter", [entry.handle, scale]); | |
| }, "LoRA adapter update was cancelled."); | |
| this._activeLoraScaleMap().set(handle, scale); | |
| } | |
| async removeLoraAdapter(handle) { | |
| const facadeHandle = loraHandleFrom(handle); | |
| return this._runExclusive( | |
| () => this._removeLoraAdapterUnlocked(facadeHandle), | |
| { kind: "lora-adapter-remove" } | |
| ); | |
| } | |
| async _removeLoraAdapterUnlocked(handle) { | |
| const entry = this._loraAdapterEntry(handle); | |
| await this._runLoraCall(async () => { | |
| if (entry.owner === this._loraOwner()) { | |
| await this._callLoraOwner("removeLoraAdapter", [entry.handle]); | |
| } | |
| }, "LoRA adapter update was cancelled."); | |
| this._activeLoraScaleMap().delete(handle); | |
| } | |
| async clearLoraAdapters() { | |
| return this._runExclusive( | |
| () => this._clearLoraAdaptersUnlocked(), | |
| { kind: "lora-adapters-clear" } | |
| ); | |
| } | |
| async _clearLoraAdaptersUnlocked() { | |
| await this._runLoraCall( | |
| () => this._callLoraOwner("clearLoraAdapters", []), | |
| "LoRA adapter update was cancelled." | |
| ); | |
| this._activeLoraScaleMap().clear(); | |
| } | |
| async tokenize(text, addSpecial = true) { | |
| return this._runExclusive( | |
| () => this._tokenizeUnlocked(text, addSpecial), | |
| { kind: "tokenize" } | |
| ); | |
| } | |
| async _tokenizeUnlocked(text, addSpecial = true) { | |
| if (!this._workerProxy) { | |
| return this._runtime.tokenize(text, addSpecial); | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| return await this._callWorker("tokenize", [text, addSpecial]); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Tokenization was cancelled."); | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback({}, error); | |
| return this._runtime.tokenize(text, addSpecial); | |
| } | |
| } | |
| async stateSaveFile(path, tokens = []) { | |
| return this._runExclusive( | |
| () => this._stateSaveFileUnlocked(path, tokens), | |
| { kind: "state-save-file" } | |
| ); | |
| } | |
| async _stateSaveFileUnlocked(path, tokens = []) { | |
| if (!this._workerProxy) { | |
| return this._runtime.stateSaveFile(path, tokens); | |
| } | |
| const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); | |
| await this._restoreWorkerModelIfMissing(); | |
| return this._callWorker("stateSaveFile", [path, normalized]); | |
| } | |
| // The capacity default is resolved inside the slot: evaluating it as a | |
| // parameter default would read runtime state before the queue was taken. | |
| async stateLoadFile(path, tokenCapacity = void 0) { | |
| return this._runExclusive( | |
| () => this._stateLoadFileUnlocked(path, tokenCapacity), | |
| { kind: "state-load-file" } | |
| ); | |
| } | |
| _resolveTokenCapacity(tokenCapacity) { | |
| let numericCapacity; | |
| try { | |
| numericCapacity = Number(tokenCapacity); | |
| } catch { | |
| return this.getContextSize(); | |
| } | |
| return numericCapacity > 0 ? Math.trunc(numericCapacity) : this.getContextSize(); | |
| } | |
| async _stateLoadFileUnlocked(path, tokenCapacity) { | |
| tokenCapacity = this._resolveTokenCapacity(tokenCapacity); | |
| if (!this._workerProxy) { | |
| return this._runtime.stateLoadFile(path, tokenCapacity); | |
| } | |
| await this._restoreWorkerModelIfMissing(); | |
| return this._callWorker("stateLoadFile", [path, tokenCapacity]); | |
| } | |
| async stateSaveBytes(tokens = []) { | |
| return this._runExclusive( | |
| () => this._stateSaveBytesUnlocked(tokens), | |
| { kind: "state-save-bytes" } | |
| ); | |
| } | |
| async _stateSaveBytesUnlocked(tokens = []) { | |
| if (!this._workerProxy) { | |
| return this._runtime.stateSaveBytes(tokens); | |
| } | |
| const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); | |
| await this._restoreWorkerModelIfMissing(); | |
| return this._callWorker("stateSaveBytes", [normalized]); | |
| } | |
| async stateLoadBytes(bytes, tokenCapacity = void 0) { | |
| return this._runExclusive( | |
| () => this._stateLoadBytesUnlocked(bytes, tokenCapacity), | |
| { kind: "state-load-bytes" } | |
| ); | |
| } | |
| async _stateLoadBytesUnlocked(bytes, tokenCapacity) { | |
| tokenCapacity = this._resolveTokenCapacity(tokenCapacity); | |
| if (!this._workerProxy) { | |
| return this._runtime.stateLoadBytes(bytes, tokenCapacity); | |
| } | |
| const normalizedBytes = toUint8Array(bytes); | |
| if (!normalizedBytes || normalizedBytes.length === 0) { | |
| throw new Error("State bytes are empty."); | |
| } | |
| await this._restoreWorkerModelIfMissing(); | |
| const transferableBytes = new Uint8Array(normalizedBytes); | |
| return this._callWorker( | |
| "stateLoadBytes", | |
| [transferableBytes, tokenCapacity], | |
| null, | |
| [transferableBytes.buffer] | |
| ); | |
| } | |
| async detokenize(tokens, special = false) { | |
| return this._runExclusive( | |
| () => this._detokenizeUnlocked(tokens, special), | |
| { kind: "detokenize" } | |
| ); | |
| } | |
| async _detokenizeUnlocked(tokens, special = false) { | |
| if (!this._workerProxy) { | |
| return this._runtime.detokenize(tokens, special); | |
| } | |
| const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []); | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| return await this._callWorker("detokenize", [normalized, special]); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Detokenization was cancelled."); | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback({}, error); | |
| return this._runtime.detokenize(normalized, special); | |
| } | |
| } | |
| async getEmbeddingCapabilities() { | |
| return this._runExclusive(async () => { | |
| try { | |
| return await this._getEmbeddingCapabilitiesUnlocked(); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Embedding capability probe was cancelled."); | |
| if (!this._workerProxy || !this._isWorkerUnusableError(error)) throw error; | |
| await this._recoverEmbeddingWorker(error); | |
| return this._runtime.getEmbeddingCapabilities(); | |
| } | |
| }, { kind: "embedding-capabilities" }); | |
| } | |
| async _getEmbeddingCapabilitiesUnlocked() { | |
| if (!this._workerProxy) { | |
| await this._ensureRememberedEmbeddingProjector(); | |
| return this._runtime.getEmbeddingCapabilities(); | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| return embeddingCapabilitiesFrom( | |
| await this._callWorker("getEmbeddingCapabilities", []), | |
| embeddingAssetVersion(this._config.workerUrl) | |
| ); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Embedding capability probe was cancelled."); | |
| if (error instanceof Error && /bridge\[method\] is not a function/.test(error.message)) { | |
| throw missingEmbeddingCapability(embeddingAssetVersion(this._config.workerUrl)); | |
| } | |
| throw error; | |
| } | |
| } | |
| async embed(text, options = {}) { | |
| const input = typeof text === "string" ? text : snapshotEmbedInput(text); | |
| return this._runExclusive( | |
| () => this._embedUnlocked(input, options), | |
| { kind: "embedding" } | |
| ); | |
| } | |
| _hasEmbeddingMediaParts(input) { | |
| return Array.isArray(input) && input.some((part) => part.type !== "text"); | |
| } | |
| async _recoverEmbeddingWorker(error) { | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback({}, error); | |
| await this._ensureRememberedEmbeddingProjector(); | |
| } | |
| async _ensureRememberedEmbeddingProjector() { | |
| if (!this._loadedMmProjUrl || !this._runtime?._modelBytes || this._runtime._mmProjPath) return; | |
| this._throwIfDisposed(); | |
| try { | |
| await this._runtime.loadMultimodalProjector(this._loadedMmProjUrl); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Embedding projector restore was cancelled."); | |
| this._emitBridgeWarn("llamadart: embedding projector restore failed; keeping the loaded model."); | |
| } | |
| this._throwIfDisposed(); | |
| } | |
| async _embedUnlocked(text, options = {}) { | |
| if (!this._workerProxy) { | |
| if (this._hasEmbeddingMediaParts(text)) await this._ensureRememberedEmbeddingProjector(); | |
| return this._runtime.embed(text, options); | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| if (Array.isArray(text)) await this._getEmbeddingCapabilitiesUnlocked(); | |
| return await this._callWorker("embed", [text, options]); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Embedding was cancelled."); | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| await this._recoverEmbeddingWorker(error); | |
| return this._runtime.embed(text, options); | |
| } | |
| } | |
| async embedBatch(texts, options = {}) { | |
| const inputs = Array.from(texts || [], (input) => typeof input === "string" ? input : snapshotEmbedInput(input)); | |
| return this._runExclusive( | |
| () => this._embedBatchUnlocked(inputs, options), | |
| { kind: "embedding-batch" } | |
| ); | |
| } | |
| async _embedBatchUnlocked(texts, options = {}) { | |
| const normalized = Array.isArray(texts) ? texts : Array.from(texts || []); | |
| if (!this._workerProxy) { | |
| if (normalized.some((input) => this._hasEmbeddingMediaParts(input))) await this._ensureRememberedEmbeddingProjector(); | |
| return this._runtime.embedBatch(normalized, options); | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| if (normalized.some(Array.isArray)) await this._getEmbeddingCapabilitiesUnlocked(); | |
| return await this._callWorker("embedBatch", [normalized, options]); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Batch embedding was cancelled."); | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| await this._recoverEmbeddingWorker(error); | |
| return this._runtime.embedBatch(normalized, options); | |
| } | |
| } | |
| async scoreNextToken(prompt, options = {}) { | |
| return this._runExclusive( | |
| () => this._scoreNextTokenUnlocked(prompt, options), | |
| { kind: "next-token-scoring" } | |
| ); | |
| } | |
| async _scoreNextTokenUnlocked(prompt, options = {}) { | |
| if (!this._workerProxy) { | |
| return this._runtime.scoreNextToken(prompt, options); | |
| } | |
| try { | |
| await this._restoreWorkerModelIfMissing(); | |
| return await this._callWorker("scoreNextToken", [prompt, options]); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Next-token scoring was cancelled."); | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| this._disableWorkerFallback(error); | |
| await this._waitForWorkerDisposal(); | |
| await this._ensureRuntimeReadyAfterWorkerFallback({}, error); | |
| return this._runtime.scoreNextToken(prompt, options); | |
| } | |
| } | |
| getModelMetadata() { | |
| if (this._workerProxy) { | |
| return { | |
| ...this._metadata || {}, | |
| "llamadart.webgpu.execution": "worker" | |
| }; | |
| } | |
| const workerReason = typeof this._workerFallbackReason === "string" && this._workerFallbackReason.length > 0 ? this._workerFallbackReason : null; | |
| return { | |
| ...this._metadata || {}, | |
| "llamadart.webgpu.execution": "main-thread", | |
| ...workerReason == null ? {} : { "llamadart.webgpu.worker_fallback_reason": workerReason } | |
| }; | |
| } | |
| // The synchronous getters answer from the facade snapshot in both modes. | |
| // Reading the direct runtime here would ccall into the singleton core outside | |
| // the queue and race whichever async operation currently owns it; the snapshot | |
| // is refreshed by _captureDirectRuntimeState() while the slot is held. | |
| getContextSize() { | |
| return this._contextSize || 0; | |
| } | |
| isGpuActive() { | |
| return this._gpuActive; | |
| } | |
| getBackendName() { | |
| return this._backendName; | |
| } | |
| setLogLevel(level) { | |
| this._throwIfDisposed(); | |
| if (Number.isFinite(level)) { | |
| this._config.logLevel = Math.max(0, Math.min(4, Math.trunc(level))); | |
| } | |
| if (this._workerProxy) { | |
| this._callWorker("setLogLevel", [level]).catch(() => { | |
| }); | |
| if (this._runtime) { | |
| this._runtime.setLogLevel(level); | |
| } | |
| return; | |
| } | |
| this._runtime.setLogLevel(level); | |
| } | |
| cancel() { | |
| if (this._lifecycleState === "disposing" || this._lifecycleState === "disposed") { | |
| return; | |
| } | |
| if (this._activeOperation) { | |
| this._cancelOperation(this._activeOperation); | |
| return; | |
| } | |
| if (this._workerProxy) { | |
| this._sendWorkerCancel(this._workerProxy); | |
| } | |
| if (this._runtime) { | |
| try { | |
| this._runtime.cancel(); | |
| } catch (_) { | |
| } | |
| } | |
| } | |
| /** | |
| * Marks the bridge disposed synchronously, then tears down on the queue so | |
| * teardown never runs underneath an operation that still owns the runtime. | |
| * Work queued before disposal is rejected with a stable lifecycle error rather | |
| * than dereferencing a torn-down runtime. Not `async`, so repeated calls hand | |
| * back the identical teardown promise. | |
| */ | |
| dispose() { | |
| if (this._disposed) { | |
| return this._disposePromise; | |
| } | |
| this._disposed = true; | |
| this._lifecycleState = "disposing"; | |
| let resolveDispose = () => { | |
| }; | |
| let rejectDispose = () => { | |
| }; | |
| this._disposePromise = new Promise((resolve, reject) => { | |
| resolveDispose = resolve; | |
| rejectDispose = reject; | |
| }); | |
| this._notifyDisposalWaiters(); | |
| const teardown = this._runExclusive( | |
| () => this._disposeUnlocked(), | |
| { allowDisposed: true, kind: "dispose" } | |
| ); | |
| teardown.then( | |
| () => resolveDispose(), | |
| (error) => rejectDispose(error) | |
| ); | |
| return this._disposePromise; | |
| } | |
| async _disposeUnlocked() { | |
| const workerProxy = this._workerProxy; | |
| this._workerProxy = null; | |
| if (workerProxy) { | |
| this._retireWorkerProxy(workerProxy); | |
| } | |
| try { | |
| if (this._retiringWorkerDisposals?.size > 0) { | |
| await this._waitForWorkerDisposal(); | |
| } | |
| if (this._runtime) { | |
| const runtime = this._runtime; | |
| this._runtime = null; | |
| await runtime.dispose(); | |
| } | |
| } finally { | |
| this._metadata = {}; | |
| this._contextSize = 0; | |
| this._gpuActive = false; | |
| this._backendName = "WASM (Prototype bridge)"; | |
| this._supportsVision = false; | |
| this._supportsAudio = false; | |
| this._loadedModelUrl = null; | |
| this._loadedModelOptions = null; | |
| this._loadedMmProjUrl = null; | |
| this._loadedDraftModel = null; | |
| this._workerFallbackReason = null; | |
| this._multimodalWorkerCpuMode = false; | |
| this._workerModelMissing = false; | |
| this._decisionHeads?.clear(); | |
| this._forgetLoraAdapters(); | |
| this._lifecycleState = "disposed"; | |
| } | |
| } | |
| async applyChatTemplate(messages, addAssistant = true, customTemplate = null) { | |
| return this._runExclusive( | |
| () => this._applyChatTemplateUnlocked(messages, addAssistant, customTemplate), | |
| { kind: "template" } | |
| ); | |
| } | |
| async _applyChatTemplateUnlocked(messages, addAssistant = true, customTemplate = null) { | |
| if (!this._workerProxy) { | |
| return this._runtime.applyChatTemplate(messages, addAssistant, customTemplate); | |
| } | |
| try { | |
| return await this._callWorker("applyChatTemplate", [messages, addAssistant, customTemplate]); | |
| } catch (error) { | |
| this._throwIfOperationCancelled(error, "Chat template operation was cancelled."); | |
| if (!this._isWorkerUnusableError(error)) { | |
| throw error; | |
| } | |
| this._disableWorkerFallback(error); | |
| return this._runtime.applyChatTemplate(messages, addAssistant, customTemplate); | |
| } | |
| } | |
| }; | |
| // js/src/worker_host.ts | |
| var textDecoder = new TextDecoder(); | |
| var bridgeWorkerHostInstalled = false; | |
| function installBridgeWorkerHost() { | |
| if (bridgeWorkerHostInstalled) { | |
| return; | |
| } | |
| if (typeof self === "undefined" || typeof self.postMessage !== "function") { | |
| throw new Error("Bridge worker host can only run inside a worker context"); | |
| } | |
| bridgeWorkerHostInstalled = true; | |
| let bridge = null; | |
| const postError = (id, error) => { | |
| let state; | |
| try { | |
| state = bridge ? snapshotBridgeState(bridge) : void 0; | |
| } catch (_) { | |
| state = void 0; | |
| } | |
| self.postMessage({ | |
| type: "error", | |
| id, | |
| message: serializeWorkerError(error), | |
| ...error instanceof EmbeddingError ? { embeddingErrorCode: error.code } : {}, | |
| state | |
| }); | |
| }; | |
| self.onmessage = async (event) => { | |
| const message = event.data || {}; | |
| const type = message.type; | |
| const id = message.id ?? 0; | |
| try { | |
| if (type === "init") { | |
| bridge = new LlamaWebGpuBridge({ | |
| ...message.config || {}, | |
| disableWorker: true | |
| }); | |
| self.postMessage({ type: "ready" }); | |
| return; | |
| } | |
| if (type !== "call") { | |
| return; | |
| } | |
| if (!bridge) { | |
| throw new Error("Bridge worker is not initialized"); | |
| } | |
| const method = String(message.method || ""); | |
| const args = Array.isArray(message.args) ? message.args : []; | |
| if (method === "loadModelFromUrl") { | |
| const url = args[0]; | |
| const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; | |
| options.progressCallback = (progress) => { | |
| self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); | |
| }; | |
| const value2 = await bridge.loadModelFromUrl(url, options); | |
| self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) }); | |
| return; | |
| } | |
| if (method === "loadDraftModel") { | |
| const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; | |
| delete options.signal; | |
| options.progressCallback = (progress) => { | |
| self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); | |
| }; | |
| const value2 = await bridge.loadDraftModel(args[0], options); | |
| self.postMessage({ type: "result", id, value: value2 }); | |
| return; | |
| } | |
| if (method === "createCompletion") { | |
| const prompt = args[0]; | |
| const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; | |
| delete options.signal; | |
| let usage = null; | |
| options.onUsage = (value3) => { | |
| usage = value3; | |
| }; | |
| const tokenEventEncoding = typeof options.tokenEventEncoding === "string" ? String(options.tokenEventEncoding || "").toLowerCase() : "bytes"; | |
| const flushMsRaw = Number(options.tokenEventFlushMs); | |
| const tokenEventFlushMs = Number.isFinite(flushMsRaw) && flushMsRaw >= 0 ? Math.max(0, Math.min(200, Math.trunc(flushMsRaw))) : 0; | |
| const flushCharsRaw = Number(options.tokenEventFlushChars); | |
| const tokenEventFlushChars = Number.isFinite(flushCharsRaw) && flushCharsRaw > 0 ? Math.max(1, Math.min(1024, Math.trunc(flushCharsRaw))) : 0; | |
| const shouldEmitCurrentText = options.emitCurrentTextOnToken === true; | |
| let pendingPieceText = ""; | |
| let pendingPieceBytes = []; | |
| let pendingPieceByteLength = 0; | |
| let pendingPieceCharLength = 0; | |
| let pendingPieceCharDecoder = tokenEventFlushChars > 0 ? new TextDecoder() : null; | |
| let pendingCurrentText = ""; | |
| let flushTimer = null; | |
| const flushTokenTextPayload = () => { | |
| if (pendingPieceText.length === 0) { | |
| return; | |
| } | |
| self.postMessage({ | |
| type: "event", | |
| id, | |
| event: "token", | |
| payload: { | |
| pieceText: pendingPieceText, | |
| currentText: shouldEmitCurrentText ? pendingCurrentText : "" | |
| } | |
| }); | |
| pendingPieceText = ""; | |
| pendingCurrentText = ""; | |
| }; | |
| const flushTokenBytePayload = () => { | |
| if (pendingPieceByteLength === 0) { | |
| return; | |
| } | |
| const piece = new Uint8Array(pendingPieceByteLength); | |
| let offset = 0; | |
| for (const chunk of pendingPieceBytes) { | |
| piece.set(chunk, offset); | |
| offset += chunk.byteLength; | |
| } | |
| self.postMessage({ | |
| type: "event", | |
| id, | |
| event: "token", | |
| payload: { | |
| piece: Array.from(piece), | |
| currentText: shouldEmitCurrentText ? pendingCurrentText : "" | |
| } | |
| }); | |
| pendingPieceBytes = []; | |
| pendingPieceByteLength = 0; | |
| pendingPieceCharLength = 0; | |
| pendingCurrentText = ""; | |
| }; | |
| const flushTokenPayload = () => { | |
| if (tokenEventEncoding === "text") { | |
| flushTokenTextPayload(); | |
| return; | |
| } | |
| flushTokenBytePayload(); | |
| }; | |
| const scheduleTokenFlush = () => { | |
| if (tokenEventFlushMs <= 0 || flushTimer != null) { | |
| return; | |
| } | |
| flushTimer = globalThis.setTimeout(() => { | |
| flushTimer = null; | |
| flushTokenPayload(); | |
| }, tokenEventFlushMs); | |
| }; | |
| options.onToken = (piece, currentText) => { | |
| if (tokenEventEncoding === "text") { | |
| const pieceText = typeof piece === "string" ? piece : textDecoder.decode(toUint8Array(piece) || new Uint8Array()); | |
| if (pieceText.length === 0) { | |
| return; | |
| } | |
| if (tokenEventFlushMs > 0) { | |
| pendingPieceText += pieceText; | |
| if (shouldEmitCurrentText) { | |
| pendingCurrentText = String(currentText || ""); | |
| } | |
| if (tokenEventFlushChars > 0 && pendingPieceText.length >= tokenEventFlushChars) { | |
| if (flushTimer != null) { | |
| globalThis.clearTimeout(flushTimer); | |
| flushTimer = null; | |
| } | |
| flushTokenTextPayload(); | |
| return; | |
| } | |
| scheduleTokenFlush(); | |
| return; | |
| } | |
| self.postMessage({ | |
| type: "event", | |
| id, | |
| event: "token", | |
| payload: { | |
| pieceText, | |
| currentText: shouldEmitCurrentText ? String(currentText || "") : "" | |
| } | |
| }); | |
| return; | |
| } | |
| const normalizedPieceBytes = toUint8Array(piece) || new Uint8Array(); | |
| if (normalizedPieceBytes.byteLength === 0) { | |
| return; | |
| } | |
| if (tokenEventFlushMs > 0) { | |
| const pieceBytes = Array.isArray(piece) ? normalizedPieceBytes : Uint8Array.from(normalizedPieceBytes); | |
| pendingPieceBytes.push(pieceBytes); | |
| pendingPieceByteLength += pieceBytes.byteLength; | |
| if (pendingPieceCharDecoder != null) { | |
| pendingPieceCharLength += pendingPieceCharDecoder.decode( | |
| pieceBytes, | |
| { stream: true } | |
| ).length; | |
| } | |
| if (shouldEmitCurrentText) { | |
| pendingCurrentText = String(currentText || ""); | |
| } | |
| if (tokenEventFlushChars > 0 && pendingPieceCharLength >= tokenEventFlushChars) { | |
| if (flushTimer != null) { | |
| globalThis.clearTimeout(flushTimer); | |
| flushTimer = null; | |
| } | |
| flushTokenBytePayload(); | |
| return; | |
| } | |
| scheduleTokenFlush(); | |
| return; | |
| } | |
| self.postMessage({ | |
| type: "event", | |
| id, | |
| event: "token", | |
| payload: { | |
| piece: Array.from(normalizedPieceBytes), | |
| currentText: shouldEmitCurrentText ? String(currentText || "") : "" | |
| } | |
| }); | |
| }; | |
| let value2; | |
| try { | |
| value2 = await bridge.createCompletion(prompt, options); | |
| } finally { | |
| if (flushTimer != null) { | |
| globalThis.clearTimeout(flushTimer); | |
| flushTimer = null; | |
| } | |
| flushTokenPayload(); | |
| } | |
| flushTokenTextPayload(); | |
| if (usage != null) { | |
| self.postMessage({ type: "event", id, event: "usage", payload: usage }); | |
| } | |
| self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) }); | |
| return; | |
| } | |
| if (method === "loadMultimodalProjector") { | |
| const value2 = await bridge.loadMultimodalProjector(args[0]); | |
| self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) }); | |
| return; | |
| } | |
| if (method === "synthesizeSpeech") { | |
| const options = args[0] && typeof args[0] === "object" ? { ...args[0] } : {}; | |
| delete options.signal; | |
| options.onProgress = (progress) => { | |
| self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); | |
| }; | |
| const value2 = await bridge.synthesizeSpeech(options); | |
| const transfers = value2?.pcm?.buffer instanceof ArrayBuffer ? [value2.pcm.buffer] : []; | |
| self.postMessage( | |
| { type: "result", id, value: value2, state: snapshotBridgeState(bridge) }, | |
| transfers | |
| ); | |
| return; | |
| } | |
| if (method === "loadDecisionHead") { | |
| const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; | |
| options.onProgress = (progress) => { | |
| self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); | |
| }; | |
| const value2 = await bridge.loadDecisionHead(args[0], options); | |
| self.postMessage({ type: "result", id, value: value2 }); | |
| return; | |
| } | |
| if (method === "loadLoraAdapter") { | |
| const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {}; | |
| delete options.signal; | |
| options.progressCallback = (progress) => { | |
| self.postMessage({ type: "event", id, event: "progress", payload: progress || {} }); | |
| }; | |
| const value2 = await bridge.loadLoraAdapter(args[0], options); | |
| self.postMessage({ type: "result", id, value: value2 }); | |
| return; | |
| } | |
| if (method === "runDecision") { | |
| const value2 = await bridge.runDecision(args[0], args[1]); | |
| const transfers = []; | |
| for (const output of Array.isArray(value2) ? value2 : []) { | |
| for (const values of [output?.logits, output?.actLogits]) { | |
| const buffer = values?.buffer; | |
| if (buffer instanceof ArrayBuffer && !transfers.includes(buffer)) { | |
| transfers.push(buffer); | |
| } | |
| } | |
| } | |
| self.postMessage({ type: "result", id, value: value2 }, transfers); | |
| return; | |
| } | |
| if (method === "unloadMultimodalProjector") { | |
| const value2 = await bridge.unloadMultimodalProjector(); | |
| self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) }); | |
| return; | |
| } | |
| if (method === "stateSaveBytes") { | |
| const value2 = await bridge.stateSaveBytes(args[0]); | |
| const transfers = value2 && value2.buffer instanceof ArrayBuffer ? [value2.buffer] : []; | |
| self.postMessage({ type: "result", id, value: value2 }, transfers); | |
| return; | |
| } | |
| if (method === "stateLoadBytes") { | |
| const value2 = await bridge.stateLoadBytes(args[0], args[1]); | |
| self.postMessage({ type: "result", id, value: value2 }); | |
| return; | |
| } | |
| if (method === "dispose") { | |
| const value2 = await bridge.dispose(); | |
| self.postMessage({ | |
| type: "result", | |
| id, | |
| value: value2, | |
| state: emptyBridgeState() | |
| }); | |
| return; | |
| } | |
| const value = await bridge[method]( | |
| ...args || [] | |
| ); | |
| self.postMessage({ type: "result", id, value }); | |
| } catch (error) { | |
| postError(id, error); | |
| } | |
| }; | |
| } | |
| function shouldAutoBootBridgeWorkerHost() { | |
| if (typeof WorkerGlobalScope === "undefined" || !(globalThis instanceof WorkerGlobalScope)) { | |
| return false; | |
| } | |
| try { | |
| const href = String(globalThis.location?.href || ""); | |
| if (!href) { | |
| return false; | |
| } | |
| const url = new URL(href); | |
| return url.searchParams.get(bridgeWorkerModeParam) === "1"; | |
| } catch (_) { | |
| return false; | |
| } | |
| } | |
| function enableBridgeWorkerHost() { | |
| installBridgeWorkerHost(); | |
| } | |
| // js/src/llama_webgpu_bridge.js | |
| if (shouldAutoBootBridgeWorkerHost()) { | |
| installBridgeWorkerHost(); | |
| } | |
| if (typeof window !== "undefined") { | |
| const browserWindow = ( | |
| /** @type {Window & typeof globalThis & { LlamaWebGpuBridge?: typeof LlamaWebGpuBridge }} */ | |
| window | |
| ); | |
| if (!browserWindow.LlamaWebGpuBridge) { | |
| browserWindow.LlamaWebGpuBridge = LlamaWebGpuBridge; | |
| } | |
| } | |
| export { | |
| EmbeddingError, | |
| LlamaWebGpuBridge, | |
| enableBridgeWorkerHost | |
| }; | |