flutter-laya-tetris / webgpu_bridge /llama_webgpu_bridge.js
leehack's picture
chore: deploy Laya Tetris 47dfa73
7f9b9f2 verified
Raw History Blame Contribute Delete
296 kB
// Generated by scripts/build_js_bridge.mjs from js/src/llama_webgpu_bridge.js.
// Do not edit this file directly; update js/src/ and run npm run build:js.
// js/src/internal/abort.ts
function createAbortError(message) {
if (typeof DOMException === "function") {
return new DOMException(message, "AbortError");
}
const error = new Error(message);
error.name = "AbortError";
return error;
}
function throwIfAborted(signal, message = "Bridge operation was cancelled.") {
if (signal?.aborted) {
throw createAbortError(message);
}
}
// js/src/internal/embedding.ts
var EmbeddingError = class extends Error {
constructor(code, message) {
super(message);
this.name = "EmbeddingError";
this.code = code;
}
};
var EMPTY_EMBEDDING_CAPABILITIES = {
apiVersion: 1,
text: false,
image: false,
audio: false
};
var EMBEDDING_MEDIA_MARKER = "<__media__>";
function embeddingAssetVersion(...sources) {
for (const source of [...sources, import.meta.url]) {
try {
const version = new URL(source || "", import.meta.url).pathname.match(/(?:^|[/@])(v?\d+\.\d+\.\d+(?:-\d+)?)(?:[/]|$)/)?.[1];
if (version) return version;
} catch (_) {
}
}
return "unversioned/local";
}
function missingEmbeddingCapability(version = embeddingAssetVersion()) {
return new EmbeddingError(
"EMBEDDING_CAPABILITY_UNAVAILABLE",
`Loaded bridge/assets version ${version} has no known media-embedding capability API version 1. Upgrade the loaded bridge assets.`
);
}
function embeddingCapabilitiesFrom(value, version = embeddingAssetVersion()) {
let parsed = value;
if (typeof value === "string") {
try {
parsed = JSON.parse(value);
} catch (_) {
throw missingEmbeddingCapability(version);
}
}
const caps = parsed;
if (!caps || caps.apiVersion !== 1 || typeof caps.text !== "boolean" || typeof caps.image !== "boolean" || typeof caps.audio !== "boolean") {
throw missingEmbeddingCapability(version);
}
return { apiVersion: 1, text: caps.text, image: caps.image, audio: caps.audio };
}
function snapshotEmbedInput(input) {
if (!Array.isArray(input)) return String(input);
if (!Array.isArray(input) || input.length === 0) {
throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding parts must be a nonempty array.");
}
const parts = Array.from(input, (part) => {
if (!part || typeof part !== "object") {
throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Invalid embedding part.");
}
if (part.type === "text") {
if (typeof part.text !== "string") {
throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding text part must contain a string.");
}
return { type: "text", text: part.text };
}
if (part.type !== "image" && part.type !== "audio") {
throw new EmbeddingError("EMBEDDING_MODALITY_UNSUPPORTED", "Embedding part modality is unsupported.");
}
if ("path" in part) {
throw new EmbeddingError("EMBEDDING_MODALITY_UNSUPPORTED", "Browser embeddings require media bytes, URL or PCM; filesystem paths are unsupported.");
}
const sourceCount = Number(part.bytes != null) + Number(part.url != null) + Number("samples" in part && part.samples != null);
if (sourceCount !== 1 || part.bytes != null && (!(part.bytes instanceof Uint8Array) || part.bytes.length === 0) || part.url != null && (typeof part.url !== "string" || part.url.length === 0)) {
throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding media requires exactly one nonempty supported source.");
}
if ("samples" in part && part.samples != null) {
if (part.type !== "audio" || !(part.samples instanceof Float32Array) || part.samples.length === 0 || !part.samples.every(Number.isFinite)) {
throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Embedding PCM must be finite mono 16 kHz Float32 samples.");
}
}
return { ...part };
});
const merged = [];
for (const part of parts) {
const previous = merged.at(-1);
if (part.type === "text" && previous?.type === "text") previous.text += part.text;
else merged.push(part);
}
if (merged.some((part) => part.type === "text" && part.text.includes(EMBEDDING_MEDIA_MARKER))) {
throw new EmbeddingError("INVALID_EMBEDDING_INPUT", "Reserved media marker in embedding text.");
}
return merged;
}
function requireEmbeddingModalities(input, caps, hasProjector) {
if (typeof input === "string") return;
if (!caps.text) {
throw new EmbeddingError("MODEL_NOT_EMBEDDING", "Loaded model does not support pooled embeddings.");
}
for (const part of input) {
if (part.type === "text") continue;
if (!hasProjector) {
throw new EmbeddingError("EMBEDDING_PROJECTOR_MISSING", "Media embeddings require a loaded projector.");
}
if (!caps[part.type]) {
throw new EmbeddingError("EMBEDDING_MODALITY_UNSUPPORTED", "Loaded projector/model does not support the requested embedding modality.");
}
}
}
function embeddingCoreError(rc, version = embeddingAssetVersion()) {
if (rc === -20) return missingEmbeddingCapability(version);
const errors = {
[-21]: ["MODEL_NOT_EMBEDDING", "Loaded model does not support this pooled embedding operation."],
[-22]: ["EMBEDDING_PROJECTOR_MISSING", "Media embeddings require a loaded projector."],
[-23]: ["EMBEDDING_INPUT_TOO_LARGE", "Expanded embedding input exceeds the allocated context, batch, sequence or single microbatch capacity."],
[-24]: ["EMBEDDING_MODALITY_UNSUPPORTED", "Loaded projector does not support the requested embedding modality or PCM rate."]
};
const [code, message] = errors[rc] || ["EMBEDDING_FAILED", "Embedding execution failed."];
return new EmbeddingError(code, message);
}
// js/src/internal/completion_options.ts
var SPECULATIVE_DECODING_STRATEGIES = Object.freeze([
"draft-simple",
"draft-eagle3",
"draft-mtp",
"draft-dflash",
"draft-dspark",
"ngram-simple",
"ngram-map-k",
"ngram-map-k4v",
"ngram-mod",
"ngram-cache"
]);
function speculativeFlags(isSupported) {
return Object.fromEntries(
SPECULATIVE_DECODING_STRATEGIES.map((strategy) => [strategy, isSupported(strategy)])
);
}
var NO_COMPLETION_CAPABILITIES = Object.freeze({
presencePenalty: false,
minP: false,
thinkingBudget: false,
speculativeDecoding: Object.freeze(speculativeFlags(() => false))
});
var MAX_NGRAM_SIZE = 65535;
var MAX_INT32 = 2147483647;
var MAX_THINKING_BUDGET_TOKENS = 2147483647;
function optionalNumber(value, name, isValid, range) {
if (value == null) {
return null;
}
if (typeof value !== "number" || !Number.isFinite(Math.fround(value)) || !isValid(value)) {
throw new RangeError(`CompletionOptions.${name} must be a finite number${range}; got ${String(value)}.`);
}
return value;
}
function tag(budget, name) {
const value = budget[name];
if (typeof value !== "string" || value.trim().length === 0) {
throw new TypeError(`CompletionOptions.thinkingBudget.${name} must be a non-empty string.`);
}
return value;
}
function thinkingBudget(value, hasMediaParts) {
if (value == null) {
return null;
}
if (typeof value !== "object") {
throw new TypeError("CompletionOptions.thinkingBudget must be an object.");
}
const budget = value;
const maxTokens = budget.maxTokens;
if (!Number.isInteger(maxTokens) || maxTokens < 0 || maxTokens > MAX_THINKING_BUDGET_TOKENS) {
throw new RangeError(
`CompletionOptions.thinkingBudget.maxTokens must be an integer from 0 to ${MAX_THINKING_BUDGET_TOKENS}; got ${String(maxTokens)}.`
);
}
const startTag = tag(budget, "startTag");
const endTag = tag(budget, "endTag");
const forcedMessage = budget.forcedMessage ?? "";
if (typeof forcedMessage !== "string") {
throw new TypeError("CompletionOptions.thinkingBudget.forcedMessage must be a string.");
}
if (hasMediaParts) {
throw new Error("CompletionOptions.thinkingBudget supports text-only prompts; remove the media parts.");
}
return { maxTokens, startTag, endTag, forcedMessage };
}
function speculativeInteger(config, name, min, max) {
const value = config[name];
if (value == null) {
return null;
}
if (!Number.isInteger(value) || value < min || value > max) {
throw new RangeError(
`CompletionOptions.speculativeDecoding.${name} must be an integer from ${min} to ${max}; got ${String(value)}.`
);
}
return value;
}
function speculativeProbability(config, name) {
const value = config[name];
if (value == null) {
return null;
}
if (typeof value !== "number" || !(value >= 0 && value <= 1)) {
throw new RangeError(
`CompletionOptions.speculativeDecoding.${name} must be a number from 0 to 1; got ${String(value)}.`
);
}
return value;
}
function ngramCacheSource(config, name) {
const value = config[name];
if (value == null) {
return null;
}
const isBytes = value instanceof ArrayBuffer || ArrayBuffer.isView(value);
if (!(typeof value === "string" || isBytes)) {
throw new TypeError(
`CompletionOptions.speculativeDecoding.${name} must be a URL string, an ArrayBuffer or a typed array.`
);
}
if ((typeof value === "string" ? value.length : value.byteLength) === 0) {
throw new TypeError(`CompletionOptions.speculativeDecoding.${name} is empty.`);
}
return value;
}
function isDraftStrategy(strategy) {
return strategy.startsWith("draft-");
}
function speculativeDecoding(value, conflicts) {
if (value == null) {
return null;
}
if (typeof value !== "object") {
throw new TypeError("CompletionOptions.speculativeDecoding must be an object.");
}
const config = value;
const requested = config.strategies;
if (!Array.isArray(requested) || requested.length === 0) {
throw new TypeError("CompletionOptions.speculativeDecoding.strategies must be a non-empty array.");
}
const strategies = [];
for (const strategy of requested) {
if (!SPECULATIVE_DECODING_STRATEGIES.includes(strategy)) {
throw new TypeError(
`CompletionOptions.speculativeDecoding.strategies has an unknown strategy ${JSON.stringify(strategy)}; expected one of ${SPECULATIVE_DECODING_STRATEGIES.join(", ")}.`
);
}
if (!strategies.includes(strategy)) {
strategies.push(strategy);
}
}
const draftTokenMax = speculativeInteger(config, "draftTokenMax", 0, MAX_INT32);
const draftTokenMin = speculativeInteger(config, "draftTokenMin", 0, MAX_INT32);
const minProbability = speculativeProbability(config, "minProbability");
const draftSplitProbability = speculativeProbability(config, "draftSplitProbability");
const ngramSizeN = speculativeInteger(config, "ngramSizeN", 1, MAX_NGRAM_SIZE);
const ngramSizeM = speculativeInteger(config, "ngramSizeM", 1, MAX_NGRAM_SIZE);
const ngramMinHits = speculativeInteger(config, "ngramMinHits", 1, MAX_NGRAM_SIZE);
const ngramMatch = speculativeInteger(config, "ngramMatch", 1, MAX_NGRAM_SIZE);
const ngramTokenMin = speculativeInteger(config, "ngramTokenMin", 0, MAX_INT32);
const ngramTokenMax = speculativeInteger(config, "ngramTokenMax", 0, MAX_INT32);
const ngramCacheStatic = ngramCacheSource(config, "ngramCacheStatic");
const ngramCacheDynamic = ngramCacheSource(config, "ngramCacheDynamic");
if (strategies.filter(isDraftStrategy).length > 1) {
throw new Error(
"CompletionOptions.speculativeDecoding can mix n-gram strategies with at most one draft-* strategy."
);
}
if (!strategies.some(isDraftStrategy) && (draftTokenMin != null || minProbability != null || draftSplitProbability != null)) {
throw new Error(
"CompletionOptions.speculativeDecoding: n-gram strategies use token history and do not support draftTokenMin, minProbability or draftSplitProbability unless a draft-* strategy is also enabled."
);
}
if ((ngramCacheStatic != null || ngramCacheDynamic != null) && !strategies.includes("ngram-cache")) {
throw new Error(
"CompletionOptions.speculativeDecoding: ngramCacheStatic and ngramCacheDynamic need the ngram-cache strategy."
);
}
if (ngramTokenMin != null && ngramTokenMax != null && ngramTokenMin > ngramTokenMax) {
throw new RangeError(
`CompletionOptions.speculativeDecoding.ngramTokenMin (${ngramTokenMin}) must not exceed ngramTokenMax (${ngramTokenMax}).`
);
}
let resolvedDraftTokenMax = 0;
for (const strategy of strategies) {
let strategyMax;
if (isDraftStrategy(strategy)) {
strategyMax = draftTokenMax ?? 3;
} else if (strategy === "ngram-mod") {
strategyMax = ngramTokenMax ?? draftTokenMax ?? 64;
} else if (strategy === "ngram-cache") {
strategyMax = draftTokenMax ?? 8;
} else {
strategyMax = ngramSizeM ?? 48;
}
resolvedDraftTokenMax = Math.max(resolvedDraftTokenMax, strategyMax);
}
if (resolvedDraftTokenMax === 0) {
resolvedDraftTokenMax = 64;
}
if ((draftTokenMin ?? 0) > resolvedDraftTokenMax) {
throw new RangeError(
`CompletionOptions.speculativeDecoding.draftTokenMin (${draftTokenMin}) must not exceed the draft token maximum (${resolvedDraftTokenMax}).`
);
}
if (conflicts.hasMediaParts) {
throw new Error("CompletionOptions.speculativeDecoding supports text-only prompts; remove the media parts.");
}
if (conflicts.hasThinkingBudget) {
throw new Error("CompletionOptions.speculativeDecoding cannot be combined with CompletionOptions.thinkingBudget.");
}
if (conflicts.hasGrammar) {
throw new Error("CompletionOptions.speculativeDecoding does not support CompletionOptions.grammar.");
}
return {
strategies,
draftTokenMax: resolvedDraftTokenMax,
draftTokenMin: draftTokenMin ?? 0,
minProbability: minProbability ?? -1,
draftSplitProbability: draftSplitProbability ?? -1,
ngramSizeN: ngramSizeN ?? 0,
ngramSizeM: ngramSizeM ?? 0,
ngramMinHits: ngramMinHits ?? 0,
ngramMatch: ngramMatch ?? 0,
ngramTokenMin: ngramTokenMin ?? -1,
ngramTokenMax: ngramTokenMax ?? (strategies.includes("ngram-mod") ? draftTokenMax ?? 0 : 0),
ngramCacheStatic,
ngramCacheDynamic
};
}
function resolveCompletionSamplingOptions(options) {
const hasMediaParts = Array.isArray(options?.parts) && options.parts.length > 0;
const budget = thinkingBudget(options?.thinkingBudget, hasMediaParts);
return {
minP: optionalNumber(options?.minP, "minP", (value) => value >= 0 && value <= 1, " from 0 to 1") ?? 0,
presencePenalty: optionalNumber(options?.presencePenalty, "presencePenalty", () => true, "") ?? 0,
thinkingBudget: budget,
speculative: speculativeDecoding(options?.speculativeDecoding, {
hasMediaParts,
hasGrammar: typeof options?.grammar === "string" && options.grammar.length > 0,
hasThinkingBudget: budget != null
})
};
}
function completionCapabilitiesFrom(raw) {
let parsed = null;
try {
parsed = JSON.parse(raw || "{}");
} catch (_) {
return { ...NO_COMPLETION_CAPABILITIES };
}
const speculative = parsed?.speculativeDecoding;
return {
presencePenalty: parsed?.presencePenalty === true,
minP: parsed?.minP === true,
thinkingBudget: parsed?.thinkingBudget === true,
speculativeDecoding: speculativeFlags((strategy) => speculative?.[strategy] === true)
};
}
function requireCompletionCapabilities(sampling, capabilities) {
const missing = [
sampling.minP !== 0 && !capabilities.minP ? "minP" : null,
sampling.presencePenalty !== 0 && !capabilities.presencePenalty ? "presencePenalty" : null,
sampling.thinkingBudget != null && !capabilities.thinkingBudget ? "thinkingBudget" : null,
sampling.speculative != null && !Object.values(capabilities.speculativeDecoding).includes(true) ? "speculativeDecoding" : null
].filter((name) => name != null);
if (missing.length > 0) {
throw new Error(
`The loaded WebGPU core does not support CompletionOptions.${missing.join(" or CompletionOptions.")}.`
);
}
}
// js/src/internal/constants.ts
var defaultModelCacheName = "llamadart-webgpu-model-cache-v1";
var BRIDGE_DISPOSED_MESSAGE = "Bridge has been disposed.";
var GENERATION_ALREADY_ACTIVE_RC = -7;
var GENERATION_ALREADY_ACTIVE_MESSAGE = "Generation is already active on this bridge runtime.";
var INVALID_GRAMMAR_ERROR_TEXT = "Failed to initialize sampler chain (invalid grammar)";
// js/src/internal/typed_values.ts
function toUint8Array(value) {
if (!value) {
return null;
}
if (value instanceof Uint8Array) {
return value;
}
if (ArrayBuffer.isView(value)) {
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
}
if (value instanceof ArrayBuffer) {
return new Uint8Array(value);
}
if (Array.isArray(value)) {
return Uint8Array.from(value.map((v) => Number(v) & 255));
}
return null;
}
function bufferSourceBytes(source) {
if (source instanceof ArrayBuffer) {
return new Uint8Array(source);
}
if (ArrayBuffer.isView(source)) {
return new Uint8Array(source.buffer, source.byteOffset, source.byteLength);
}
return null;
}
function toFloat32Array(value) {
if (!value) {
return null;
}
if (value instanceof Float32Array) {
return value;
}
if (ArrayBuffer.isView(value)) {
return new Float32Array(
value.buffer,
value.byteOffset,
Math.floor(value.byteLength / Float32Array.BYTES_PER_ELEMENT)
);
}
if (value instanceof ArrayBuffer) {
return new Float32Array(value);
}
if (Array.isArray(value)) {
return Float32Array.from(value.map((v) => Number(v) || 0));
}
return null;
}
function isInt32(value) {
return typeof value === "number" && Number.isInteger(value) && value >= -2147483648 && value <= 2147483647;
}
// js/src/internal/decision.ts
var DECISION_API_VERSION = 1;
var DECISION_WORKER_TIMEOUT_PER_SEQUENCE_MS = 60 * 1e3;
function decisionHandleFrom(handle) {
if (!isInt32(handle) || handle <= 0) {
throw new TypeError(`Decision head handle must be a positive integer, got ${String(handle)}.`);
}
return handle;
}
function decisionIntegerList(value, label) {
const isList = Array.isArray(value) || ArrayBuffer.isView(value) && !(value instanceof DataView);
if (!isList) {
throw new TypeError(`${label} must be an array or typed array of integers.`);
}
const list = value;
const out = new Int32Array(list.length);
for (let index = 0; index < list.length; index += 1) {
const item = list[index];
if (!isInt32(item)) {
throw new TypeError(`${label}[${index}] is ${String(item)}; expected a 32-bit integer.`);
}
out[index] = item;
}
return out;
}
function normalizeDecisionSequences(sequences) {
if (!Array.isArray(sequences)) {
throw new TypeError("Decision sequences must be an array.");
}
return sequences.map((sequence, index) => {
const label = `Decision sequence ${index}`;
if (!sequence || typeof sequence !== "object") {
throw new TypeError(`${label} must be an object with tokens, markers and questionType.`);
}
const tokens = decisionIntegerList(sequence.tokens, `${label} tokens`);
const markers = decisionIntegerList(sequence.markers, `${label} markers`);
const questionType = sequence.questionType;
if (!isInt32(questionType)) {
throw new TypeError(
`${label} questionType is ${String(questionType)}; expected 0 (choice), 1 (score) or 2 (noul).`
);
}
return { tokens, markers, questionType };
});
}
function encodeDecisionSequences(sequences) {
const encoded = normalizeDecisionSequences(sequences);
let words = 1;
for (const sequence of encoded) {
words += 3 + sequence.tokens.length + sequence.markers.length;
}
const bytes = new Uint8Array(words * 4);
const view = new DataView(bytes.buffer);
let offset = 0;
const put = (value) => {
view.setInt32(offset, value, true);
offset += 4;
};
put(encoded.length);
for (const sequence of encoded) {
put(sequence.questionType);
put(sequence.tokens.length);
put(sequence.markers.length);
sequence.tokens.forEach(put);
sequence.markers.forEach(put);
}
return bytes;
}
function decodeDecisionOutputs(bytes, count) {
const malformed = () => new Error("Decision output from the WebGPU core is malformed.");
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
let offset = 0;
const floats = (length) => {
const values = new Float32Array(length);
for (let index = 0; index < length; index += 1) {
values[index] = view.getFloat32(offset, true);
offset += 4;
}
return values;
};
const outputs = [];
for (let index = 0; index < count; index += 1) {
if (bytes.byteLength - offset < 8) {
throw malformed();
}
const logitCount = view.getInt32(offset, true);
const actCount = view.getInt32(offset + 4, true);
offset += 8;
if (logitCount < 0 || actCount < 0 || (logitCount + actCount) * 4 > bytes.byteLength - offset) {
throw malformed();
}
outputs.push({ logits: floats(logitCount), actLogits: floats(actCount) });
}
if (offset !== bytes.byteLength) {
throw malformed();
}
return outputs;
}
// js/src/internal/logging.ts
function logLevelForName(level) {
switch (level) {
case "debug":
return 0;
case "log":
case "info":
return 1;
case "warn":
return 2;
case "error":
return 3;
default:
return 1;
}
}
function logThresholdForConfiguredLevel(level) {
switch (level) {
case 0:
return 99;
case 1:
return 0;
case 2:
return 1;
case 3:
return 2;
case 4:
return 3;
default:
return 1;
}
}
// js/src/internal/lora.ts
var LORA_API_VERSION = 1;
var LORA_LOAD_ABORT_MESSAGE = "LoRA adapter load was cancelled.";
function loraHandleFrom(handle) {
if (!isInt32(handle) || handle <= 0) {
throw new TypeError(`LoRA adapter handle must be a positive integer, got ${String(handle)}.`);
}
return handle;
}
function loraScaleFrom(scale) {
if (typeof scale !== "number" || !Number.isFinite(scale)) {
throw new TypeError(`LoRA adapter scale must be a finite number, got ${String(scale)}.`);
}
if (!Number.isFinite(Math.fround(scale))) {
throw new RangeError(`LoRA adapter scale ${String(scale)} is outside the 32-bit float range.`);
}
return scale;
}
function staleLoraAdapterError(handle) {
return new Error(
`LoRA adapter ${handle} is not loaded; its model was unloaded or replaced. Load the adapter again.`
);
}
// js/src/internal/model_source.ts
function cloneModelSource(source) {
if (Array.isArray(source)) {
return source.slice();
}
return String(source || "").trim();
}
function hasModelSource(source) {
return Array.isArray(source) ? source.length > 0 : typeof source === "string" && source.length > 0;
}
function basenameFromUrl(url) {
try {
const parsed = new URL(url, typeof window !== "undefined" ? window.location.href : void 0);
const pathname = parsed.pathname || "";
const name = pathname.split("/").pop() || "model.gguf";
return name.includes("?") ? name.split("?")[0] : name;
} catch (_) {
const parts = String(url).split("/");
return parts[parts.length - 1] || "model.gguf";
}
}
function normalizeAbsoluteUrl(url) {
try {
return new URL(url, typeof window !== "undefined" ? window.location.href : void 0).toString();
} catch (_) {
return String(url);
}
}
function parseSplitShardPattern(fileName) {
if (typeof fileName !== "string" || fileName.length === 0) {
return null;
}
const match = fileName.match(/^(.*)-(\d{4,6})-of-(\d{4,6})\.gguf$/i);
if (!match) {
return null;
}
const total = Number(match[3]);
if (!Number.isInteger(total) || total < 2 || total > 512) {
return null;
}
return {
prefix: match[1],
width: Math.max(match[2].length, match[3].length),
total
};
}
function expandModelShardUrls(modelUrlOrUrls) {
if (Array.isArray(modelUrlOrUrls)) {
return modelUrlOrUrls.map((value) => String(value || "").trim()).filter((value) => value.length > 0).map((value) => normalizeAbsoluteUrl(value));
}
const source = String(modelUrlOrUrls || "").trim();
if (source.length === 0) {
return [];
}
try {
const parsed = new URL(source, typeof window !== "undefined" ? window.location.href : void 0);
const pathname = parsed.pathname || "";
const slash = pathname.lastIndexOf("/");
const dirPath = slash >= 0 ? pathname.slice(0, slash + 1) : "";
const fileName = slash >= 0 ? pathname.slice(slash + 1) : pathname;
const split = parseSplitShardPattern(fileName);
if (!split) {
return [parsed.toString()];
}
const totalShardId = String(split.total).padStart(split.width, "0");
const urls = [];
for (let shardIndex = 1; shardIndex <= split.total; shardIndex += 1) {
const shardId = String(shardIndex).padStart(split.width, "0");
parsed.pathname = `${dirPath}${split.prefix}-${shardId}-of-${totalShardId}.gguf`;
urls.push(parsed.toString());
}
return urls;
} catch (_) {
return [source];
}
}
// js/src/internal/core_loader.ts
function normalizeFactory(moduleExport) {
if (typeof moduleExport === "function") {
return moduleExport;
}
if (moduleExport && typeof moduleExport.default === "function") {
return moduleExport.default;
}
if (moduleExport && typeof moduleExport.createLlamaWebGpuCoreModule === "function") {
return moduleExport.createLlamaWebGpuCoreModule;
}
throw new Error("Unable to resolve llama_webgpu_core factory function");
}
async function importCoreFactory(moduleUrl) {
const exportedModule = await import(moduleUrl);
return normalizeFactory(exportedModule);
}
// js/src/internal/parse.ts
function parsePositiveInteger(value) {
const numeric = Number(value);
if (!Number.isFinite(numeric) || numeric <= 0) {
return 0;
}
return Math.trunc(numeric);
}
function parseInteger(value, fallback = 0) {
const numeric = Number(value);
if (!Number.isFinite(numeric)) {
return fallback;
}
return Math.trunc(numeric);
}
function parseBooleanFlag(value, fallback = false) {
if (typeof value === "boolean") {
return value;
}
if (typeof value === "number" && Number.isFinite(value)) {
return value !== 0;
}
return fallback;
}
function parseOptionalBooleanFlag(value) {
if (typeof value === "boolean") {
return value ? 1 : 0;
}
if (typeof value === "number" && Number.isFinite(value)) {
return value !== 0 ? 1 : 0;
}
return -1;
}
function parseEnumValue(value, allowed, fallback) {
const parsed = parseInteger(value, fallback);
return allowed.includes(parsed) ? parsed : fallback;
}
function parsePositiveNumber(value) {
const numeric = Number(value);
return Number.isFinite(numeric) && numeric > 0 ? numeric : 0;
}
// js/src/internal/download.ts
function hasReadableResponseStream(response) {
return !!(response && response.body && typeof response.body.getReader === "function");
}
function sumProgressValues(values) {
let total = 0;
for (const value of values || []) {
const numeric = Number(value);
if (Number.isFinite(numeric) && numeric > 0) {
total += numeric;
}
}
return total;
}
function parseTotalFromContentRangeHeader(contentRangeHeader) {
if (typeof contentRangeHeader !== "string" || contentRangeHeader.length === 0) {
return 0;
}
const slash = contentRangeHeader.lastIndexOf("/");
if (slash < 0 || slash + 1 >= contentRangeHeader.length) {
return 0;
}
return parsePositiveInteger(contentRangeHeader.slice(slash + 1));
}
function inferResponseTotalBytes(response, loadedFallback = 0) {
if (!response || !response.headers) {
return parsePositiveInteger(loadedFallback);
}
const linkedSize = parsePositiveInteger(response.headers.get("x-linked-size"));
if (linkedSize > 0) {
return linkedSize;
}
const contentRangeTotal = parseTotalFromContentRangeHeader(
response.headers.get("content-range")
);
if (contentRangeTotal > 0) {
return contentRangeTotal;
}
const contentLength = parsePositiveInteger(response.headers.get("content-length"));
if (contentLength > 0) {
return contentLength;
}
return parsePositiveInteger(loadedFallback);
}
function isRetryableStreamNetworkError(error) {
const text = String(error || "").toLowerCase();
return text.includes("network error") || text.includes("failed to fetch") || text.includes("networkerror") || text.includes("err_network_io_suspended") || text.includes("the network connection was lost") || text.includes("connection reset") || text.includes("timeout") || text.includes("timed out");
}
async function readStreamChunkWithTimeout(reader, timeoutMs, label = "stream read") {
const resolvedTimeout = Number(timeoutMs);
if (!Number.isFinite(resolvedTimeout) || resolvedTimeout <= 0) {
return reader.read();
}
let timeoutHandle = null;
try {
return await Promise.race([
reader.read(),
new Promise((_, reject) => {
timeoutHandle = globalThis.setTimeout(() => {
reject(new Error(`${label} timeout (${resolvedTimeout}ms)`));
}, resolvedTimeout);
})
]);
} finally {
if (timeoutHandle != null) {
globalThis.clearTimeout(timeoutHandle);
}
}
}
async function drainResponseWithProgress(response, progressCallback, options = {}) {
const total = Number(response.headers.get("content-length")) || 0;
const chunkTimeoutMs = parsePositiveInteger(options.chunkTimeoutMs);
if (!response.body || typeof response.body.getReader !== "function") {
const bytes = new Uint8Array(await response.arrayBuffer());
if (typeof progressCallback === "function") {
progressCallback({ loaded: bytes.byteLength, total: total || bytes.byteLength });
}
return bytes.byteLength;
}
const reader = response.body.getReader();
let loaded = 0;
let lastBucket = -1;
while (true) {
const { done, value } = await readStreamChunkWithTimeout(
reader,
chunkTimeoutMs,
"response drain read"
);
if (done) {
break;
}
if (!value || value.length === 0) {
continue;
}
loaded += value.length;
if (typeof progressCallback === "function") {
const effectiveTotal = total || loaded;
const bucket = effectiveTotal > 0 ? Math.floor(loaded / effectiveTotal * 100) : -1;
if (bucket > lastBucket) {
lastBucket = bucket;
progressCallback({ loaded, total: effectiveTotal });
}
}
}
if (typeof progressCallback === "function") {
progressCallback({ loaded, total: total || loaded });
}
return loaded;
}
function ensureFsDirectory(fs, dirPath) {
const slash = dirPath.lastIndexOf("/");
const parent = slash > 0 ? dirPath.slice(0, slash) : "/";
const name = dirPath.slice(slash + 1);
const exists = () => {
try {
const entries = fs.readdir(parent);
return Array.isArray(entries) && entries.includes(name);
} catch (_) {
return false;
}
};
if (exists()) {
return;
}
try {
fs.mkdir(dirPath);
} catch (error) {
if (!exists()) {
throw error;
}
}
}
function unlinkFsFile(fs, filePath) {
try {
fs.unlink(filePath);
return true;
} catch (_) {
return false;
}
}
async function writeResponseToFsFileWithProgress(response, fs, filePath, progressCallback, writeOptions = {}) {
const total = parsePositiveInteger(writeOptions.totalBytes) || inferResponseTotalBytes(response, 0);
const useBigIntPosition = writeOptions.useBigIntPosition === true;
const startOffset = parsePositiveInteger(writeOptions.startOffset);
const preservePartialOnError = writeOptions.preservePartialOnError === true;
const appendMode = startOffset > 0;
const chunkTimeoutMs = parsePositiveInteger(writeOptions.chunkTimeoutMs);
const signal = writeOptions.signal || null;
const abortMessage = writeOptions.abortMessage || "Model transfer was cancelled.";
throwIfAborted(signal, abortMessage);
if (!appendMode) {
unlinkFsFile(fs, filePath);
}
if (!response.body || typeof response.body.getReader !== "function") {
throwIfAborted(signal, abortMessage);
const bytes = new Uint8Array(await response.arrayBuffer());
throwIfAborted(signal, abortMessage);
if (appendMode) {
const stream2 = fs.open(filePath, "r+");
try {
throwIfAborted(signal, abortMessage);
const position = useBigIntPosition ? BigInt(startOffset) : startOffset;
fs.write(stream2, bytes, 0, bytes.length, position);
} finally {
fs.close(stream2);
}
} else {
fs.writeFile(filePath, bytes);
}
const finalLoaded2 = startOffset + bytes.byteLength;
if (typeof progressCallback === "function") {
progressCallback({ loaded: finalLoaded2, total: total || finalLoaded2 });
}
return finalLoaded2;
}
const reader = response.body.getReader();
const openMode = appendMode ? "r+" : "w";
const stream = fs.open(filePath, openMode);
let loaded = 0;
let lastBucket = -1;
let writePosition = startOffset;
let writePositionBigInt = null;
try {
while (true) {
throwIfAborted(signal, abortMessage);
const { done, value } = await readStreamChunkWithTimeout(
reader,
chunkTimeoutMs,
"response file read"
);
if (done) {
throwIfAborted(signal, abortMessage);
break;
}
if (!value || value.length === 0) {
continue;
}
const chunk = value.slice ? value.slice() : new Uint8Array(value);
throwIfAborted(signal, abortMessage);
if (useBigIntPosition) {
if (writePositionBigInt == null) {
writePositionBigInt = BigInt(startOffset);
}
fs.write(stream, chunk, 0, chunk.length, writePositionBigInt);
writePositionBigInt += BigInt(chunk.length);
} else {
fs.write(stream, chunk, 0, chunk.length, writePosition);
writePosition += chunk.length;
}
loaded += chunk.length;
if (typeof progressCallback === "function") {
const effectiveLoaded = startOffset + loaded;
const effectiveTotal = total || effectiveLoaded;
const bucket = effectiveTotal > 0 ? Math.floor(effectiveLoaded / effectiveTotal * 100) : -1;
if (bucket > lastBucket) {
lastBucket = bucket;
progressCallback({ loaded: effectiveLoaded, total: effectiveTotal });
}
}
}
} catch (error) {
if (signal?.aborted && error?.name !== "AbortError") {
error = createAbortError(abortMessage);
}
try {
if (error && typeof error === "object") {
error.llamadartLoadedBytes = startOffset + loaded;
error.llamadartFilePath = filePath;
}
} catch (_) {
}
try {
fs.close(stream);
} catch (_) {
}
try {
await reader.cancel?.();
} catch (_) {
}
if (!preservePartialOnError || signal?.aborted || error?.name === "AbortError") {
try {
fs.unlink(filePath);
} catch (_) {
}
}
throw error;
}
fs.close(stream);
const finalLoaded = startOffset + loaded;
if (typeof progressCallback === "function") {
progressCallback({ loaded: finalLoaded, total: total || finalLoaded });
}
return finalLoaded;
}
// js/src/internal/environment.ts
function isSafariUserAgent(userAgent) {
if (typeof userAgent !== "string" || userAgent.length === 0) {
return false;
}
const hasSafariToken = /Safari\//.test(userAgent);
const hasOtherBrowserToken = /(Chrome|Chromium|CriOS|Edg|OPR|Firefox|FxiOS)\//.test(userAgent);
return hasSafariToken && !hasOtherBrowserToken;
}
function isCrossOriginIsolatedRuntime() {
try {
if (typeof globalThis.crossOriginIsolated === "boolean") {
return globalThis.crossOriginIsolated;
}
} catch (_) {
}
return true;
}
// js/src/internal/image.ts
async function decodeImageBytesToRgb(bytes, options = {}) {
const sourceBytes = toUint8Array(bytes);
if (!sourceBytes || sourceBytes.length === 0) {
return null;
}
if (typeof createImageBitmap !== "function" || typeof Blob !== "function") {
return null;
}
const maxPixelsCandidate = Number(options.maxPixels);
const maxPixels = Number.isFinite(maxPixelsCandidate) && maxPixelsCandidate > 0 ? Math.max(65536, Math.min(33554432, Math.trunc(maxPixelsCandidate))) : 0;
const maxEdgeCandidate = Number(options.maxEdge);
const maxEdge = Number.isFinite(maxEdgeCandidate) && maxEdgeCandidate > 0 ? Math.max(64, Math.min(16384, Math.trunc(maxEdgeCandidate))) : 0;
if (maxPixels <= 0 && maxEdge <= 0) {
return null;
}
let bitmap = null;
try {
const mimeType = typeof options.mimeType === "string" && options.mimeType.length > 0 ? options.mimeType : "image/png";
const blob = new Blob([sourceBytes], { type: mimeType });
bitmap = await createImageBitmap(blob);
const sourceWidth = Math.max(1, Math.trunc(Number(bitmap.width) || 0));
const sourceHeight = Math.max(1, Math.trunc(Number(bitmap.height) || 0));
let scale = 1;
if (maxPixels > 0) {
const sourcePixels = sourceWidth * sourceHeight;
if (sourcePixels > maxPixels) {
scale = Math.min(scale, Math.sqrt(maxPixels / sourcePixels));
}
}
if (maxEdge > 0) {
const sourceLongest = Math.max(sourceWidth, sourceHeight);
if (sourceLongest > maxEdge) {
scale = Math.min(scale, maxEdge / sourceLongest);
}
}
const width = Math.max(1, Math.round(sourceWidth * scale));
const height = Math.max(1, Math.round(sourceHeight * scale));
let canvas = null;
let context = null;
if (typeof OffscreenCanvas === "function") {
canvas = new OffscreenCanvas(width, height);
context = canvas.getContext("2d", {
alpha: false,
willReadFrequently: true
});
}
if (!context && typeof document !== "undefined" && typeof document.createElement === "function") {
canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
context = canvas.getContext("2d", {
alpha: false,
willReadFrequently: true
});
}
if (!context) {
return null;
}
context.drawImage(bitmap, 0, 0, width, height);
let encodedBytes = null;
if (canvas && "convertToBlob" in canvas && typeof canvas.convertToBlob === "function") {
const encodedBlob = await canvas.convertToBlob({ type: "image/png" });
if (encodedBlob) {
encodedBytes = new Uint8Array(await encodedBlob.arrayBuffer());
}
}
if (!encodedBytes && canvas && "toBlob" in canvas && typeof canvas.toBlob === "function" && typeof Promise === "function") {
const encodedBlob = await new Promise((resolve) => {
canvas.toBlob((value) => {
resolve(value || null);
}, "image/png");
});
if (encodedBlob) {
encodedBytes = new Uint8Array(await encodedBlob.arrayBuffer());
}
}
if (!encodedBytes || encodedBytes.length === 0) {
return null;
}
return {
bytes: encodedBytes,
width,
height,
sourceWidth,
sourceHeight,
resized: width !== sourceWidth || height !== sourceHeight
};
} catch (_) {
return null;
} finally {
try {
bitmap?.close?.();
} catch (_) {
}
}
}
// js/src/internal/next_token_scores.ts
function nextTokenCandidateIds(value) {
if (value == null) {
return [];
}
const isList = Array.isArray(value) || ArrayBuffer.isView(value) && !(value instanceof DataView);
if (!isList) {
throw new TypeError("Next-token candidates must be an array or typed array of token ids.");
}
return Array.from(value, (item, index) => {
if (!isInt32(item)) {
throw new TypeError(`Next-token candidates[${index}] is ${String(item)}; expected a 32-bit integer.`);
}
return item;
});
}
function scoredTokensFrom(entries) {
if (!Array.isArray(entries)) {
return [];
}
return entries.map((entry) => ({
token: Number(entry?.token),
bytes: Uint8Array.from(Array.isArray(entry?.bytes) ? entry.bytes : []),
logprob: typeof entry?.logprob === "number" ? entry.logprob : -Infinity
}));
}
// js/src/internal/text.ts
function buildPromptFromMessages(messages, addAssistant) {
const lines = [];
for (const msg of messages || []) {
const role = String(msg?.role ?? "user");
const content = String(msg?.content ?? "");
lines.push(`${role}: ${content}`);
}
if (addAssistant) {
lines.push("assistant: ");
}
return lines.join("\n");
}
function looksLikeCorruptedGeneration(text) {
if (typeof text !== "string" || text.length === 0) {
return false;
}
const normalized = text.trim();
if (normalized.length === 0) {
return false;
}
const unusedTokens = text.match(/<unused\d+>/g) || [];
if (unusedTokens.length >= 4) {
return true;
}
const tokenLikeTags = text.match(/<[^>]{1,40}>/g) || [];
if (tokenLikeTags.length >= 8) {
return true;
}
const compact = text.replace(/\s+/g, "");
if (compact.length === 0) {
return false;
}
const tagRun = compact.match(/(?:<[^>]{2,32}>){6,}/);
if (tagRun) {
return true;
}
const alphaNum = (normalized.match(/[A-Za-z0-9]/g) || []).length;
const printable = (normalized.match(/[\x20-\x7E]/g) || []).length;
const angleBrackets = (normalized.match(/[<>]/g) || []).length;
const alphaNumRatio = alphaNum / normalized.length;
const printableRatio = printable / normalized.length;
const bracketRatio = angleBrackets / normalized.length;
if (normalized.length >= 24 && printableRatio > 0.95 && alphaNumRatio < 0.18) {
return true;
}
if (normalized.length >= 24 && bracketRatio > 0.25) {
return true;
}
return false;
}
function trimUnstableUtf8Tail(text) {
if (typeof text !== "string" || text.length === 0) {
return "";
}
let end = text.length;
while (end > 0 && text.charCodeAt(end - 1) === 65533) {
end -= 1;
}
if (end > 0) {
const tail = text.charCodeAt(end - 1);
if (tail >= 55296 && tail <= 56319) {
end -= 1;
}
}
return end === text.length ? text : text.slice(0, end);
}
// js/src/runtime.ts
var textEncoder = new TextEncoder();
var LlamaWebGpuBridgeRuntime = class {
constructor(config = {}) {
this._config = config;
this._core = null;
this._backendLabels = [];
this._gpuActive = false;
this._modelPath = null;
this._modelPaths = [];
this._modelBytes = 0;
this._mmProjPath = null;
this._mmSupportsVision = false;
this._mmSupportsAudio = false;
this._mediaFileCounter = 0;
this._stateFileCounter = 0;
this._decisionFileCounter = 0;
this._loraFileCounter = 0;
this._loraAdaptersLoaded = false;
this._stagedMediaPaths = [];
this._nCtx = 4096;
this._abortRequested = false;
this._textToSpeechActive = false;
this._textToSpeechDone = null;
this._resolveTextToSpeechDone = null;
this._runtimeNotes = [];
this._threadPoolSizeHint = Number(config.threadPoolSize) > 0 ? Math.max(1, Math.trunc(Number(config.threadPoolSize))) : null;
const requestedThreads = Number(config.threads) > 0 ? Number(config.threads) : this._resolveAutoThreadCount();
this._threads = this._capThreadsToPool(requestedThreads);
const requestedThreadsBatch = Number(config.threadsBatch) > 0 ? Number(config.threadsBatch) : this._threads;
this._threadsBatch = this._capThreadsToPool(
requestedThreadsBatch,
{ noteTag: "threads_batch_capped_pool" }
);
this._nBatch = Number(config.nBatch) > 0 ? Math.max(32, Math.trunc(Number(config.nBatch))) : 0;
this._nUbatch = Number(config.nUbatch) > 0 ? Math.max(32, Math.trunc(Number(config.nUbatch))) : 0;
this._nGpuLayers = Number.isFinite(config.nGpuLayers) ? Number(config.nGpuLayers) : -1;
this._nSeqMax = 0;
this._useMmap = false;
this._useMlock = false;
this._flashAttention = -1;
this._cacheTypeK = 1;
this._cacheTypeV = 1;
this._kvUnified = -1;
this._ropeFrequencyBase = 0;
this._ropeFrequencyScale = 0;
this._splitMode = -1;
this._mainGpu = -1;
this._loadMtp = false;
this._speculativeRollbackTokenMax = 0;
this._isSafari = isSafariUserAgent(this._config.userAgent ?? globalThis.navigator?.userAgent ?? "");
this._coreVariant = "uninitialized";
this._preferMemory64 = this._config.preferMemory64 !== false;
this._modelSource = "network";
this._modelCacheState = "disabled";
this._modelCacheName = defaultModelCacheName;
this._loadedModelUrl = null;
this._mmProjSourceUrl = null;
this._draftModel = null;
this._speculativeFileCounter = 0;
this._suppressedWarmupWarningCount = 0;
this._didReportWarmupWarningSuppression = false;
this._remoteFetchThresholdBytes = Number(config.remoteFetchThresholdBytes) > 0 ? Number(config.remoteFetchThresholdBytes) : 1900 * 1024 * 1024;
this._remoteFetchChunkBytes = Number(config.remoteFetchChunkBytes) > 0 ? Number(config.remoteFetchChunkBytes) : 16 * 1024 * 1024;
this._mediaMaxImagePixels = Number(config.mediaMaxImagePixels) > 0 ? Math.max(65536, Math.min(33554432, Math.trunc(Number(config.mediaMaxImagePixels)))) : 1024 * 1024;
this._mediaMaxImageEdge = Number(config.mediaMaxImageEdge) > 0 ? Math.max(64, Math.min(16384, Math.trunc(Number(config.mediaMaxImageEdge)))) : 1280;
this._disableImageDownscale = config.disableImageDownscale === true;
this._activeTransferAbortController = null;
this._lastCoreErrorText = "";
this._lastCoreErrorHint = "";
this._logLevel = Number.isFinite(config.logLevel) ? Math.max(0, Math.min(4, Math.trunc(config.logLevel))) : 2;
}
static supportsSafariAdaptiveGpu = true;
_pushRuntimeNote(note) {
if (typeof note !== "string" || note.length === 0) {
return;
}
if (!Array.isArray(this._runtimeNotes)) {
this._runtimeNotes = [];
}
if (!this._runtimeNotes.includes(note)) {
this._runtimeNotes.push(note);
}
}
_detectThreadPoolSizeFromCore() {
if (!this._coreSupportsPthreads()) {
return 1;
}
const core = this._core;
if (!core || typeof core !== "object") {
return null;
}
try {
if (typeof core.ccall === "function") {
const compiledPoolSize = Number(
core.ccall("llamadart_webgpu_pthread_pool_size", "number", [], [])
);
if (Number.isFinite(compiledPoolSize) && compiledPoolSize > 0) {
return Math.max(1, Math.trunc(compiledPoolSize));
}
}
} catch (_) {
}
try {
const pThread = core.PThread;
if (!pThread || typeof pThread !== "object") {
return null;
}
const unused = Array.isArray(pThread.unusedWorkers) ? pThread.unusedWorkers.length : 0;
const running = Array.isArray(pThread.runningWorkers) ? pThread.runningWorkers.length : 0;
const total = unused + running;
return total > 0 ? total : null;
} catch (_) {
return null;
}
}
_coreSupportsPthreads() {
const core = this._core;
if (!core || typeof core !== "object") {
return false;
}
try {
if (typeof core.ccall === "function") {
const compiledWithPthreads = Number(
core.ccall("llamadart_webgpu_supports_pthreads", "number", [], [])
);
if (Number.isFinite(compiledWithPthreads)) {
return compiledWithPthreads !== 0;
}
}
} catch (_) {
}
try {
const wasmBuffer = core.wasmMemory?.buffer;
if (typeof SharedArrayBuffer === "function" && wasmBuffer instanceof SharedArrayBuffer) {
return true;
}
} catch (_) {
}
try {
const heapBuffer = core.HEAP8?.buffer || core.HEAPU8?.buffer;
if (typeof SharedArrayBuffer === "function" && heapBuffer instanceof SharedArrayBuffer) {
return true;
}
} catch (_) {
}
try {
const pThread = core.PThread;
if (!pThread || typeof pThread !== "object") {
return false;
}
return Array.isArray(pThread.unusedWorkers) || Array.isArray(pThread.runningWorkers) || typeof pThread.allocateUnusedWorker === "function";
} catch (_) {
return false;
}
}
_syncThreadPoolSizeHintFromCore() {
const detected = this._detectThreadPoolSizeFromCore();
if (!Number.isFinite(detected) || detected <= 0) {
return;
}
this._threadPoolSizeHint = Math.max(1, Math.trunc(detected));
}
_resolveAutoThreadCount() {
const hardwareThreads = Number(globalThis.navigator?.hardwareConcurrency);
if (Number.isFinite(hardwareThreads) && hardwareThreads > 0) {
return Math.max(1, Math.min(8, Math.trunc(hardwareThreads)));
}
return 4;
}
_capThreadsToPool(candidate, { noteTag = "threads_capped_pool" } = {}) {
let resolved = Number(candidate);
if (!Number.isFinite(resolved) || resolved <= 0) {
resolved = 1;
}
resolved = Math.max(1, Math.trunc(resolved));
const poolSize = Number(this._threadPoolSizeHint);
if (Number.isFinite(poolSize) && poolSize > 0 && resolved > poolSize) {
if (noteTag) {
this._pushRuntimeNote(`${noteTag}:${poolSize}`);
}
return poolSize;
}
return resolved;
}
_isVerboseWarmupWarning(text) {
const lowered = String(text || "").toLowerCase();
if (lowered.length === 0) {
return false;
}
if (lowered.includes("warmup:")) {
return true;
}
if (lowered.includes("please report this on github as an issue")) {
return true;
}
if (lowered.includes("warning: ref:")) {
return true;
}
if (lowered.includes("github.com/ggml-org/llama.cpp/pull/")) {
return true;
}
if (lowered.includes("****************")) {
return true;
}
return false;
}
_emitSuppressedWarmupWarningSummaryIfNeeded() {
if (this._suppressedWarmupWarningCount <= 0) {
return;
}
if (this._logLevel <= 2) {
this._emitLogger(
"log",
`info: suppressed ${this._suppressedWarmupWarningCount} verbose warmup log lines (set bridge/runtime log level to Debug to inspect full warmup trace).`
);
}
this._suppressedWarmupWarningCount = 0;
}
_shouldAttemptGenerationRecovery(errorText, options = {}, generated = 0) {
if (generated > 0) {
return false;
}
if (this._nGpuLayers <= 0) {
return false;
}
if (this._coreVariant !== "wasm64") {
return false;
}
if (!hasModelSource(this._loadedModelUrl)) {
return false;
}
if (options._llamadartGenerationRecoveryAttempted === true) {
return false;
}
if (Array.isArray(options.parts) && options.parts.length > 0) {
return false;
}
const lowered = String(errorText || "").toLowerCase();
if (lowered.includes("failed to decode")) {
return true;
}
if (lowered.includes("failed to compute graph")) {
return true;
}
if (lowered.includes("ggml_backend_sched_graph_compute_async failed")) {
return true;
}
return false;
}
_isContextLimitGenerationError(errorText = "") {
const lowered = [
String(errorText || ""),
String(this._lastCoreErrorText || ""),
String(this._lastCoreErrorHint || "")
].join(" ").toLowerCase();
return lowered.includes("failed to find a memory slot for batch") || lowered.includes("failed to prepare attention batches") || lowered.includes("context overflow") || lowered.includes("context full") || lowered.includes("kv cache full") || lowered.includes("insufficient kv") || lowered.includes("no kv slot");
}
async _recoverGenerationWithCpuFallback(options = {}) {
const modelUrl = cloneModelSource(this._loadedModelUrl);
if (!hasModelSource(modelUrl)) {
return false;
}
if (this._loraAdaptersLoaded) {
this._runtimeNotes.push("generation_recovery_cpu_skipped_lora");
return false;
}
this._runtimeNotes.push("generation_recovery_cpu_attempt");
this._emitLogger(
"warn",
"warning: generation failed on wasm64/WebGPU; retrying by reloading model with CPU fallback for stability."
);
const previousPreferMemory64 = this._preferMemory64;
const previousMMProjSourceUrl = this._mmProjSourceUrl;
const previousDraftModel = this._draftModel;
const resolvedLoadState = this._resolvedLoadState();
try {
this._preferMemory64 = false;
await this.loadModelFromUrl(modelUrl, {
nCtx: this._nCtx,
nThreads: this._threads,
nGpuLayers: 0,
useCache: true,
forceRemoteFetchBackend: false,
remoteFetchChunkBytes: this._resolveRemoteFetchChunkBytes(options),
safariGpuProbe: false,
signal: options.signal
}, resolvedLoadState);
if (typeof previousMMProjSourceUrl === "string" && previousMMProjSourceUrl.length > 0) {
try {
await this.loadMultimodalProjector(previousMMProjSourceUrl);
} catch (_) {
this._runtimeNotes.push("generation_recovery_mmproj_reload_failed");
}
}
if (previousDraftModel) {
try {
await this.loadDraftModel(previousDraftModel.url, previousDraftModel.options);
} catch (_) {
this._runtimeNotes.push("generation_recovery_draft_reload_failed");
}
}
this._runtimeNotes.push("generation_recovery_cpu_applied");
return true;
} catch (_) {
this._runtimeNotes.push("generation_recovery_cpu_failed");
return false;
} finally {
this._preferMemory64 = previousPreferMemory64;
}
}
_loggerFor(level) {
const logger = this._config?.logger;
const fallback = typeof console !== "undefined" ? console : null;
if (logger && typeof logger[level] === "function") {
return logger[level].bind(logger);
}
if (!fallback) {
return () => {
};
}
if (typeof fallback[level] === "function") {
return fallback[level].bind(fallback);
}
if (typeof fallback.log === "function") {
return fallback.log.bind(fallback);
}
return () => {
};
}
_shouldEmitLoggerLevel(level) {
const current = Number(this._logLevel);
if (!Number.isFinite(current) || current < 0) {
return true;
}
const threshold = logThresholdForConfiguredLevel(
Math.max(0, Math.min(4, Math.trunc(current)))
);
if (threshold > 3) {
return false;
}
return logLevelForName(level) >= threshold;
}
_emitLogger(level, message) {
if (!this._shouldEmitLoggerLevel(level)) {
return;
}
try {
this._loggerFor(level)(message);
} catch (_) {
}
}
_classifyCoreErrorLine(text) {
const trimmed = String(text ?? "").trim();
if (trimmed.length === 0) {
return "ignore";
}
if (this._isVerboseWarmupWarning(trimmed)) {
return "warmup";
}
const lowered = trimmed.toLowerCase();
if (lowered.startsWith("warning") || lowered.startsWith("warn:") || lowered.includes(" warning:")) {
return "warn";
}
if (lowered.startsWith("error") || lowered.startsWith("err:") || lowered.includes(" error:") || lowered.includes("failed") || lowered.includes("exception") || lowered.includes("abort") || lowered.includes("fatal") || lowered.includes("out of memory") || lowered.includes("invalid")) {
return "error";
}
return "info";
}
_applyCoreLogLevel() {
if (!this._core) {
return;
}
try {
this._core.ccall(
"llamadart_webgpu_set_log_level",
null,
["number"],
[this._logLevel]
);
} catch (_) {
}
}
_coreErrorMessage(prefix, fallbackCode = 0) {
try {
const err = this._core?.ccall("llamadart_webgpu_last_error", "string", [], []);
if (err) {
return `${prefix}: ${err}`;
}
} catch (_) {
}
return `${prefix} (code=${fallbackCode})`;
}
_resolveCacheName(options = {}) {
if (typeof options.cacheName === "string" && options.cacheName.trim().length > 0) {
return options.cacheName.trim();
}
if (typeof this._config.cacheName === "string" && this._config.cacheName.trim().length > 0) {
return this._config.cacheName.trim();
}
return defaultModelCacheName;
}
_isWorkerRuntime() {
return typeof WorkerGlobalScope !== "undefined" && globalThis instanceof WorkerGlobalScope;
}
_resolveRemoteFetchThresholdBytes(options = {}) {
const candidate = Number(options.remoteFetchThresholdBytes);
if (Number.isFinite(candidate) && candidate > 0) {
return Math.trunc(candidate);
}
return Math.trunc(this._remoteFetchThresholdBytes);
}
_resolveRemoteFetchChunkBytes(options = {}) {
const candidate = Number(options.remoteFetchChunkBytes);
if (Number.isFinite(candidate) && candidate > 0) {
return Math.max(16 * 1024, Math.trunc(candidate));
}
return Math.max(16 * 1024, Math.trunc(this._remoteFetchChunkBytes));
}
_canUseRemoteFetchBackend(options = {}) {
if (options.forceRemoteFetchBackend === false) {
return false;
}
if (!this._isWorkerRuntime()) {
return false;
}
if (options.forceRemoteFetchBackend === true) {
return true;
}
return this._config.allowAutoRemoteFetchBackend === true;
}
async _tryHeadContentLength(url) {
function parseSizeFromHeaders(headers) {
if (!headers) {
return 0;
}
const length = Number(headers.get("content-length")) || 0;
if (length > 0) {
return length;
}
const linkedSize = Number(headers.get("x-linked-size")) || 0;
if (linkedSize > 0) {
return linkedSize;
}
const contentRange = String(headers.get("content-range") || "");
const slash = contentRange.lastIndexOf("/");
if (slash >= 0 && slash + 1 < contentRange.length) {
const total = Number(contentRange.slice(slash + 1)) || 0;
if (total > 0) {
return total;
}
}
return 0;
}
try {
const response = await this._fetchWithTimeout(url, {
method: "HEAD",
cache: "no-store"
}, 45e3);
if (!response.ok) {
throw new Error("HEAD request failed");
}
const size = parseSizeFromHeaders(response.headers);
if (size > 0) {
return size;
}
} catch (_) {
}
try {
const probe = await this._fetchWithTimeout(url, {
headers: {
Range: "bytes=0-0"
},
cache: "no-store"
}, 45e3);
if (!(probe.ok || probe.status === 206)) {
return null;
}
const size = parseSizeFromHeaders(probe.headers);
if (size > 0) {
this._runtimeNotes.push("model_fetch_size_probe");
return size;
}
} catch (_) {
}
return null;
}
async _resolveRemoteFetchUrl(url) {
function parseSizeFromHeaders(headers) {
if (!headers) {
return 0;
}
const contentRange = String(headers.get("content-range") || "");
const slash = contentRange.lastIndexOf("/");
if (slash >= 0 && slash + 1 < contentRange.length) {
const total = Number(contentRange.slice(slash + 1)) || 0;
if (total > 0) {
return total;
}
}
const linkedSize = Number(headers.get("x-linked-size")) || 0;
if (linkedSize > 0) {
return linkedSize;
}
const length = Number(headers.get("content-length")) || 0;
if (length > 0) {
return length;
}
return 0;
}
try {
const probe = await this._fetchWithTimeout(url, {
headers: {
Range: "bytes=0-0"
},
cache: "no-store"
}, 45e3);
if (!(probe.ok || probe.status === 206)) {
return null;
}
const resolvedUrl = typeof probe.url === "string" && probe.url.length > 0 ? probe.url : null;
const sizeBytes = parseSizeFromHeaders(probe.headers);
return {
resolvedUrl,
sizeBytes: sizeBytes > 0 ? sizeBytes : null
};
} catch (_) {
return null;
}
}
_resolveNativeLoadOptions(options = {}) {
this._nSeqMax = parsePositiveInteger(options.nSeqMax);
this._useMmap = parseBooleanFlag(options.useMmap, false);
this._useMlock = parseBooleanFlag(options.useMlock, false);
this._flashAttention = parseEnumValue(options.flashAttention, [-1, 0, 1], -1);
this._cacheTypeK = parseEnumValue(options.cacheTypeK, [1, 2, 8], 1);
this._cacheTypeV = parseEnumValue(options.cacheTypeV, [1, 2, 8], 1);
this._kvUnified = parseOptionalBooleanFlag(options.kvUnified);
this._ropeFrequencyBase = parsePositiveNumber(options.ropeFrequencyBase);
this._ropeFrequencyScale = parsePositiveNumber(options.ropeFrequencyScale);
this._splitMode = parseEnumValue(options.splitMode, [0, 1, 2, 3], -1);
this._mainGpu = parseInteger(options.mainGpu, -1);
if (this._mainGpu < 0) {
this._mainGpu = -1;
}
this._loadMtp = parseBooleanFlag(options.loadMtp, false);
this._speculativeRollbackTokenMax = parsePositiveInteger(options.speculativeRollbackTokenMax);
const wantsQuantizedKvCache = this._cacheTypeK !== 1 || this._cacheTypeV !== 1;
if (this._flashAttention === 0 && wantsQuantizedKvCache) {
throw new Error(
"Non-F16 KV cache requires flashAttention to be auto or enabled."
);
}
if (this._flashAttention === -1 && wantsQuantizedKvCache) {
this._flashAttention = 1;
this._runtimeNotes.push("flash_attention:auto_enabled_for_kv_cache");
}
if (this._kvUnified < 0 && this._nSeqMax > 1) {
this._kvUnified = 1;
this._runtimeNotes.push("kv_unified:auto_enabled_for_sequences");
}
}
_nativeLoadOptionValues() {
return [
this._nSeqMax,
this._useMmap ? 1 : 0,
this._useMlock ? 1 : 0,
this._flashAttention,
this._cacheTypeK,
this._cacheTypeV,
this._kvUnified,
this._ropeFrequencyBase,
this._ropeFrequencyScale,
this._splitMode,
this._mainGpu,
this._loadMtp ? 1 : 0,
this._speculativeRollbackTokenMax
];
}
_nativeLoadOptionTypes() {
return this._nativeLoadOptionValues().map(() => "number");
}
_resolvedLoadState() {
return {
_nCtx: this._nCtx,
_threads: this._threads,
_threadsBatch: this._threadsBatch,
_nBatch: this._nBatch,
_nUbatch: this._nUbatch,
_nSeqMax: this._nSeqMax,
_useMmap: this._useMmap,
_useMlock: this._useMlock,
_flashAttention: this._flashAttention,
_cacheTypeK: this._cacheTypeK,
_cacheTypeV: this._cacheTypeV,
_kvUnified: this._kvUnified,
_ropeFrequencyBase: this._ropeFrequencyBase,
_ropeFrequencyScale: this._ropeFrequencyScale,
_splitMode: this._splitMode,
_mainGpu: this._mainGpu,
_loadMtp: this._loadMtp,
_speculativeRollbackTokenMax: this._speculativeRollbackTokenMax
};
}
async _tryLoadModelFromRemoteFetchBackend(core, url, options = {}) {
if (!this._canUseRemoteFetchBackend(options)) {
return { loaded: false, sizeBytes: null };
}
const thresholdBytes = this._resolveRemoteFetchThresholdBytes(options);
const chunkBytes = this._resolveRemoteFetchChunkBytes(options);
const forceRemote = options.forceRemoteFetchBackend === true;
let sizeBytes = Number(options.modelBytesHint);
if (!Number.isFinite(sizeBytes) || sizeBytes <= 0) {
sizeBytes = await this._tryHeadContentLength(url);
}
if (!forceRemote) {
if (Number.isFinite(sizeBytes) && sizeBytes > 0 && sizeBytes < thresholdBytes) {
this._runtimeNotes.push("model_fetch_backend_skipped_small");
return { loaded: false, sizeBytes };
}
if (!Number.isFinite(sizeBytes) || sizeBytes <= 0) {
this._runtimeNotes.push("model_fetch_backend_size_unknown");
this._runtimeNotes.push("model_fetch_backend_unknown_size_attempt");
}
}
this._runtimeNotes.push("model_fetch_backend_attempt");
this._runtimeNotes.push(`model_fetch_chunk:${chunkBytes}`);
this._lastCoreErrorHint = "";
let remoteFetchUrl = url;
const resolvedProbe = await this._resolveRemoteFetchUrl(url);
if (resolvedProbe?.resolvedUrl) {
remoteFetchUrl = resolvedProbe.resolvedUrl;
if (remoteFetchUrl !== url) {
this._runtimeNotes.push("model_fetch_backend_resolved_url");
}
}
if ((!Number.isFinite(sizeBytes) || sizeBytes <= 0) && Number.isFinite(resolvedProbe?.sizeBytes) && resolvedProbe.sizeBytes > 0) {
sizeBytes = resolvedProbe.sizeBytes;
}
try {
globalThis.__llamadartFetchBackendLastError = null;
} catch (_) {
}
if (typeof options.progressCallback === "function") {
options.progressCallback({ loaded: 0, total: Number.isFinite(sizeBytes) ? sizeBytes : 0 });
}
try {
const rc = Number(
await core.ccall(
"llamadart_webgpu_load_model_from_url",
"number",
[
"string",
"number",
"number",
"number",
"number",
"number",
"number",
"number",
...this._nativeLoadOptionTypes()
],
[
remoteFetchUrl,
this._nCtx,
this._threads,
this._threadsBatch,
this._nBatch,
this._nUbatch,
this._nGpuLayers,
chunkBytes,
...this._nativeLoadOptionValues()
],
{ async: true }
)
);
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Fetch-backed model load failed", rc));
}
this._modelSource = "network-fetch";
this._modelPath = null;
this._modelBytes = Number.isFinite(sizeBytes) && sizeBytes > 0 ? Math.trunc(sizeBytes) : 1;
this._runtimeNotes.push("model_source_fetch_backend");
if (typeof options.progressCallback === "function") {
const resolved = Number.isFinite(sizeBytes) && sizeBytes > 0 ? sizeBytes : 1;
options.progressCallback({ loaded: resolved, total: resolved });
}
return {
loaded: true,
sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null
};
} catch (error) {
const text = String(error || "").toLowerCase();
if (text.includes("aborted(native code called abort())")) {
try {
const stderrText = String(
this._lastCoreErrorHint || this._lastCoreErrorText || ""
).trim();
if (stderrText.length > 0) {
const token = stderrText.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, "");
if (token.length > 0) {
this._runtimeNotes.push(`model_core_stderr:${token}`);
}
}
} catch (_) {
}
try {
const fetchError = String(
globalThis.__llamadartFetchBackendLastError || ""
).trim();
if (fetchError.length > 0) {
const token = fetchError.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, "");
if (token.length > 0) {
this._runtimeNotes.push(`model_fetch_js_error:${token}`);
}
}
} catch (_) {
}
try {
const stats = globalThis.__llamadartFetchBackendStats;
if (stats && typeof stats === "object") {
const reads = Number(stats.reads) || 0;
const getSize = Number(stats.getSize) || 0;
const ranges = Number(stats.ranges) || 0;
const fallbacks = Number(stats.wholeFileFallbacks) || 0;
const errors = Number(stats.errors) || 0;
this._runtimeNotes.push(
`model_fetch_stats:r${reads}_s${getSize}_q${ranges}_f${fallbacks}_e${errors}`
);
}
} catch (_) {
}
try {
const coreError = String(
this._core?.ccall("llamadart_webgpu_last_error", "string", [], []) || ""
).trim();
if (coreError.length > 0) {
const token = coreError.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, "");
if (token.length > 0) {
this._runtimeNotes.push(`model_fetch_core_error:${token}`);
}
}
} catch (_) {
}
this._runtimeNotes.push("model_fetch_backend_abort");
throw error;
}
if (text.includes("fetch-backed model load failed")) {
this._runtimeNotes.push("model_fetch_backend_failed");
return {
loaded: false,
sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null
};
}
if (text.includes("load_model_from_url") && (text.includes("not found") || text.includes("undefined symbol") || text.includes("missing function"))) {
this._runtimeNotes.push("model_fetch_backend_unavailable");
return {
loaded: false,
sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null
};
}
if (text.includes("worker-thread bridge runtime")) {
this._runtimeNotes.push("model_fetch_backend_requires_worker");
return {
loaded: false,
sizeBytes: Number.isFinite(sizeBytes) ? sizeBytes : null
};
}
throw error;
}
}
_beginTransferAbortController() {
if (typeof AbortController !== "function") {
this._activeTransferAbortController = null;
return null;
}
if (this._activeTransferAbortController) {
try {
this._activeTransferAbortController.abort();
} catch (_) {
}
}
const controller = new AbortController();
this._activeTransferAbortController = controller;
return controller;
}
_clearTransferAbortController(controller) {
if (this._activeTransferAbortController === controller) {
this._activeTransferAbortController = null;
}
}
_resolveFetchTimeoutMs(options = {}, defaultTimeoutMs = 18e4) {
const configured = Number(options.fetchTimeoutMs);
if (Number.isFinite(configured) && configured > 0) {
return Math.max(1e4, Math.min(18e5, Math.trunc(configured)));
}
return defaultTimeoutMs;
}
_resolveStreamChunkTimeoutMs(options = {}, defaultTimeoutMs = 9e4) {
const configured = Number(options.streamChunkTimeoutMs);
if (Number.isFinite(configured) && configured > 0) {
return Math.max(5e3, Math.min(3e5, Math.trunc(configured)));
}
return defaultTimeoutMs;
}
_resolveCoreInitTimeoutMs() {
const configured = Number(this._config.coreInitTimeoutMs);
if (!Number.isFinite(configured) || configured <= 0) {
return 9e4;
}
return Math.max(1e4, Math.min(6e5, Math.trunc(configured)));
}
_resolveMediaImageMaxPixels(options = {}) {
if (options.disableImageDownscale === true || this._disableImageDownscale) {
return 0;
}
const configured = Number(options.mediaMaxImagePixels);
if (Number.isFinite(configured) && configured > 0) {
return Math.max(65536, Math.min(33554432, Math.trunc(configured)));
}
return this._mediaMaxImagePixels;
}
_resolveMediaImageMaxEdge(options = {}) {
if (options.disableImageDownscale === true || this._disableImageDownscale) {
return 0;
}
const configured = Number(options.mediaMaxImageEdge);
if (Number.isFinite(configured) && configured > 0) {
return Math.max(64, Math.min(16384, Math.trunc(configured)));
}
return this._mediaMaxImageEdge;
}
async _fetchWithTimeout(url, init = {}, timeoutMs = 0) {
const resolvedTimeout = Number(timeoutMs);
if (!Number.isFinite(resolvedTimeout) || resolvedTimeout <= 0) {
return fetch(url, init);
}
if (typeof AbortController !== "function") {
return Promise.race([
fetch(url, init),
new Promise((_, reject) => {
globalThis.setTimeout(
() => reject(new Error(`fetch timeout (${resolvedTimeout}ms)`)),
resolvedTimeout
);
})
]);
}
const timeoutController = new AbortController();
const externalSignal = init.signal;
let didTimeout = false;
let timeoutHandle = null;
const onExternalAbort = () => {
try {
timeoutController.abort();
} catch (_) {
}
};
if (externalSignal && typeof externalSignal.addEventListener === "function") {
externalSignal.addEventListener("abort", onExternalAbort, { once: true });
if (externalSignal.aborted) {
onExternalAbort();
}
}
timeoutHandle = globalThis.setTimeout(() => {
didTimeout = true;
try {
timeoutController.abort();
} catch (_) {
}
}, resolvedTimeout);
try {
return await fetch(url, {
...init,
signal: timeoutController.signal
});
} catch (error) {
if (didTimeout) {
throw new Error(`fetch timeout (${resolvedTimeout}ms)`);
}
throw error;
} finally {
if (timeoutHandle != null) {
globalThis.clearTimeout(timeoutHandle);
}
if (externalSignal && typeof externalSignal.removeEventListener === "function") {
externalSignal.removeEventListener("abort", onExternalAbort);
}
}
}
_unlinkDecisionFile(path) {
if (!this._core || typeof path !== "string" || path.length === 0) {
return;
}
try {
this._core.FS.unlink(path);
} catch (_) {
}
}
_deleteFsFile(path) {
if (!this._core || typeof path !== "string" || path.length === 0) {
return false;
}
return unlinkFsFile(this._core.FS, path);
}
_releaseModelFiles() {
const paths = Array.isArray(this._modelPaths) && this._modelPaths.length > 0 ? [...this._modelPaths] : this._modelPath ? [this._modelPath] : [];
let removed = 0;
for (const path of paths) {
if (this._deleteFsFile(path)) {
removed += 1;
}
}
this._modelPaths = [];
return removed;
}
// A load replaces the current model. Free it, its projector and their FS files
// before downloading the next model so both never occupy the wasm heap at once.
// The core refuses while a generation or speech synthesis is still active,
// which leaves the current model untouched.
_releaseLoadedModel(core) {
const projectorPath = this._mmProjPath;
const hasModelFiles = Array.isArray(this._modelPaths) && this._modelPaths.length > 0;
if (this._modelBytes <= 0 && !projectorPath && !hasModelFiles) {
return false;
}
const rc = Number(core.ccall("llamadart_webgpu_free_model", "number", [], []));
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to release the loaded model", rc));
}
this._loraAdaptersLoaded = false;
this._clearStagedMediaFiles();
this._deleteFsFile(projectorPath);
this._releaseModelFiles();
this._modelPath = null;
this._modelBytes = 0;
this._modelSource = "network";
this._mmProjPath = null;
this._mmSupportsVision = false;
this._mmSupportsAudio = false;
this._draftModel = null;
this._runtimeNotes.push("previous_model_released");
return true;
}
async _getCachedModelResponse(url, options = {}) {
let useCache = options.useCache !== false;
const forceRefresh = options.force === true;
const requireReadableStream = options.requireReadableStream === true;
const requestHeaders = options.requestHeaders && typeof options.requestHeaders === "object" ? options.requestHeaders : null;
if (requestHeaders && Object.keys(requestHeaders).length > 0) {
useCache = false;
}
const fetchOptions = {
...options.signal ? { signal: options.signal } : {},
...requestHeaders ? { headers: requestHeaders } : {}
};
const fetchTimeoutMs = this._resolveFetchTimeoutMs(options, 18e4);
this._modelSource = "network";
this._modelCacheState = useCache ? "unavailable" : "disabled";
this._modelCacheName = this._resolveCacheName(options);
if (!useCache) {
const response = await this._fetchWithTimeout(url, {
cache: "no-store",
...fetchOptions
}, fetchTimeoutMs);
this._modelCacheState = "disabled";
if (requireReadableStream) {
this._runtimeNotes.push(
hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream"
);
}
return response;
}
if (!globalThis.caches || typeof globalThis.caches.open !== "function") {
this._modelCacheState = "unavailable";
const response = await this._fetchWithTimeout(url, fetchOptions, fetchTimeoutMs);
if (requireReadableStream) {
this._runtimeNotes.push(
hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream"
);
}
return response;
}
const cacheKey = normalizeAbsoluteUrl(url);
try {
const cache = await globalThis.caches.open(this._modelCacheName);
const cached = forceRefresh ? null : await cache.match(cacheKey);
if (cached) {
const cacheHasStream = hasReadableResponseStream(cached);
if (!requireReadableStream || cacheHasStream) {
this._modelSource = "cache";
this._modelCacheState = "hit";
this._runtimeNotes.push("model_cache_hit");
if (requireReadableStream) {
this._runtimeNotes.push("model_cache_stream");
}
return cached;
}
this._runtimeNotes.push("model_cache_hit_no_stream");
this._modelSource = "network";
this._modelCacheState = "refresh";
const refreshed = await this._fetchWithTimeout(url, {
cache: "no-store",
...fetchOptions
}, fetchTimeoutMs);
if (refreshed.ok) {
try {
await cache.put(cacheKey, refreshed.clone());
this._modelCacheState = "stored";
this._runtimeNotes.push("model_cache_stored");
} catch (_) {
this._modelCacheState = "store_failed";
this._runtimeNotes.push("model_cache_store_failed");
}
}
this._runtimeNotes.push(
hasReadableResponseStream(refreshed) ? "model_network_stream" : "model_network_no_stream"
);
return refreshed;
}
this._modelCacheState = forceRefresh ? "refresh" : "miss";
const response = await this._fetchWithTimeout(url, fetchOptions, fetchTimeoutMs);
if (response.ok) {
try {
await cache.put(cacheKey, response.clone());
this._modelCacheState = "stored";
this._runtimeNotes.push("model_cache_stored");
} catch (_) {
this._modelCacheState = "store_failed";
this._runtimeNotes.push("model_cache_store_failed");
}
}
if (requireReadableStream) {
this._runtimeNotes.push(
hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream"
);
}
return response;
} catch (_) {
this._modelCacheState = "error";
this._runtimeNotes.push("model_cache_error");
const response = await this._fetchWithTimeout(url, fetchOptions, fetchTimeoutMs);
if (requireReadableStream) {
this._runtimeNotes.push(
hasReadableResponseStream(response) ? "model_network_stream" : "model_network_no_stream"
);
}
return response;
}
}
async prefetchModelToCache(url, options = {}) {
const useCache = options.useCache !== false;
this._modelSource = "network";
this._modelCacheState = useCache ? "unavailable" : "disabled";
this._modelCacheName = this._resolveCacheName(options);
const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null;
const modelUrls = expandModelShardUrls(url);
if (modelUrls.length === 0) {
throw new Error("Model URL is empty.");
}
if (modelUrls.length > 1) {
this._runtimeNotes.push(`model_split_cache_prefetch:${modelUrls.length}`);
}
const shardLoaded = new Array(modelUrls.length).fill(0);
const shardTotals = new Array(modelUrls.length).fill(0);
const emitAggregateProgress = () => {
if (!progressCallback) {
return;
}
const loaded = sumProgressValues(shardLoaded);
const total = sumProgressValues(shardTotals);
progressCallback({
loaded,
total: total > 0 ? total : loaded
});
};
const controller = this._beginTransferAbortController();
try {
const fetchOptions = {
cache: "no-store",
...controller?.signal ? { signal: controller.signal } : {}
};
const fetchTimeoutMs = this._resolveFetchTimeoutMs(options, 18e4);
const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs(options, 9e4);
const cache = useCache && globalThis.caches && typeof globalThis.caches.open === "function" ? await globalThis.caches.open(this._modelCacheName) : null;
if (!useCache || !cache) {
this._modelCacheState = useCache ? "unavailable" : "disabled";
}
for (let shardIndex = 0; shardIndex < modelUrls.length; shardIndex += 1) {
const shardUrl = modelUrls[shardIndex];
const cacheKey = normalizeAbsoluteUrl(shardUrl);
let response;
let headerTotal = 0;
if (cache) {
const cached = options.force === true ? null : await cache.match(cacheKey);
if (cached) {
this._modelSource = "cache";
this._modelCacheState = "hit";
this._runtimeNotes.push("model_cache_hit");
headerTotal = Number(cached.headers.get("content-length")) || 0;
const resolved = headerTotal > 0 ? headerTotal : 1;
shardLoaded[shardIndex] = resolved;
shardTotals[shardIndex] = resolved;
emitAggregateProgress();
continue;
}
this._modelSource = "network";
this._modelCacheState = options.force === true ? "refresh" : "miss";
}
response = await this._fetchWithTimeout(shardUrl, fetchOptions, fetchTimeoutMs);
if (!response.ok) {
throw new Error(
`Failed to prefetch model shard: ${response.status} ${response.statusText}`
);
}
headerTotal = Number(response.headers.get("content-length")) || 0;
if (headerTotal > 0) {
shardTotals[shardIndex] = headerTotal;
}
const putPromise = cache ? cache.put(cacheKey, response.clone()).then(() => ({ ok: true, error: null })).catch((error) => ({ ok: false, error })) : null;
await drainResponseWithProgress(
response,
progressCallback ? (progress) => {
const loaded = Number(progress?.loaded) || 0;
const total = Number(progress?.total) || 0;
shardLoaded[shardIndex] = loaded;
if (total > 0) {
shardTotals[shardIndex] = total;
}
emitAggregateProgress();
} : null,
{ chunkTimeoutMs }
);
const finalLoaded = shardLoaded[shardIndex] > 0 ? shardLoaded[shardIndex] : shardTotals[shardIndex] > 0 ? shardTotals[shardIndex] : headerTotal;
shardLoaded[shardIndex] = finalLoaded;
if (finalLoaded > 0 && shardTotals[shardIndex] < finalLoaded) {
shardTotals[shardIndex] = finalLoaded;
}
emitAggregateProgress();
if (putPromise) {
const putResult = await putPromise;
if (putResult.ok) {
this._modelCacheState = "stored";
this._runtimeNotes.push("model_cache_stored");
} else {
const putErrorText = String(putResult.error || "").toLowerCase();
if (putErrorText.includes("abort") || putErrorText.includes("cancel")) {
throw putResult.error || new Error("Model cache prefetch was aborted.");
}
this._modelCacheState = "store_failed";
this._runtimeNotes.push("model_cache_store_failed");
throw new Error("Failed to store prefetched model in browser cache.");
}
}
}
if (modelUrls.length > 1) {
this._runtimeNotes.push("model_split_cache_prefetched");
}
return 1;
} catch (error) {
this._modelCacheState = "error";
const text = String(error || "").toLowerCase();
if (text.includes("abort")) {
this._runtimeNotes.push("model_cache_prefetch_aborted");
} else if (text.includes("quota") || text.includes("storage")) {
this._runtimeNotes.push("model_cache_quota_exceeded");
} else {
this._runtimeNotes.push("model_cache_error");
}
throw error;
} finally {
this._clearTransferAbortController(controller);
}
}
async evictModelFromCache(url, options = {}) {
this._modelCacheName = this._resolveCacheName(options);
const modelUrls = expandModelShardUrls(url);
if (modelUrls.length === 0) {
return false;
}
if (!globalThis.caches || typeof globalThis.caches.open !== "function") {
this._modelCacheState = "unavailable";
return false;
}
try {
const cache = await globalThis.caches.open(this._modelCacheName);
let removedCount = 0;
for (const modelUrl of modelUrls) {
const cacheKey = normalizeAbsoluteUrl(modelUrl);
const removed = await cache.delete(cacheKey);
if (removed) {
removedCount += 1;
}
}
const removedAny = removedCount > 0;
this._modelCacheState = removedAny ? "evicted" : "miss";
if (removedAny) {
this._runtimeNotes.push("model_cache_evicted");
}
if (modelUrls.length > 1) {
this._runtimeNotes.push(`model_split_cache_evicted:${removedCount}/${modelUrls.length}`);
}
return removedAny;
} catch (_) {
this._modelCacheState = "error";
this._runtimeNotes.push("model_cache_error");
return false;
}
}
async _ensureCore() {
if (this._core) {
this._applyCoreLogLevel();
return this._core;
}
const candidates = [];
if (this._config.coreModuleFactory) {
candidates.push({
variant: "custom",
factoryPromise: Promise.resolve(this._config.coreModuleFactory),
wasmUrl: this._config.wasmUrl
});
} else {
if (this._preferMemory64 && typeof this._config.coreModuleUrlMem64 === "string" && this._config.coreModuleUrlMem64.length > 0) {
candidates.push({
variant: "wasm64",
factoryPromise: importCoreFactory(this._config.coreModuleUrlMem64),
wasmUrl: this._config.wasmUrlMem64 || this._config.wasmUrl
});
}
candidates.push({
variant: "wasm32",
factoryPromise: importCoreFactory(this._config.coreModuleUrl ?? "./llama_webgpu_core.js"),
wasmUrl: this._config.wasmUrl
});
}
let lastError = null;
for (const candidate of candidates) {
if (candidate.variant === "wasm64") {
this._runtimeNotes.push("core_mem64_attempt");
}
try {
const moduleFactory = await candidate.factoryPromise;
const initTimeoutMs = this._resolveCoreInitTimeoutMs();
this._core = await Promise.race([
moduleFactory({
locateFile: (path, prefix) => {
if (path.endsWith(".wasm") && candidate.wasmUrl) {
return candidate.wasmUrl;
}
return `${prefix}${path}`;
},
print: (msg) => {
this._emitLogger("log", msg);
},
printErr: (msg) => {
const text = String(msg ?? "");
this._lastCoreErrorText = text;
const trimmed = text.trim();
if (trimmed.length > 0) {
const loweredTrimmed = trimmed.toLowerCase();
const isGenericAbort = loweredTrimmed === "aborted(native code called abort())" || loweredTrimmed === "native code called abort()" || loweredTrimmed === "aborted";
if (!isGenericAbort) {
this._lastCoreErrorHint = trimmed;
}
}
const classification = this._classifyCoreErrorLine(text);
if (classification === "ignore") {
return;
}
if (classification === "warmup") {
if (this._logLevel >= 2) {
this._suppressedWarmupWarningCount += 1;
if (!this._didReportWarmupWarningSuppression) {
this._didReportWarmupWarningSuppression = true;
if (this._logLevel <= 2) {
this._emitLogger(
"log",
"info: suppressing verbose warmup op logs; set bridge/runtime log level to Debug to inspect all warmup details."
);
}
}
this._pushRuntimeNote("warmup_warning_suppressed");
return;
}
this._emitLogger("log", trimmed.length > 0 ? trimmed : text);
return;
}
if (classification === "warn") {
this._emitLogger("warn", trimmed.length > 0 ? trimmed : text);
return;
}
if (classification === "info") {
this._emitLogger("log", trimmed.length > 0 ? trimmed : text);
return;
}
this._emitLogger("error", trimmed.length > 0 ? trimmed : text);
},
onAbort: (reason) => {
const text = String(reason ?? "").trim();
if (text.length > 0) {
const token = text.slice(0, 120).replace(/[\s;=]+/g, "_").replace(/[^a-zA-Z0-9._:-]/g, "");
if (token.length > 0) {
this._runtimeNotes.push(`core_abort:${token}`);
}
this._emitLogger("error", `core abort: ${text}`);
} else {
this._runtimeNotes.push("core_abort");
this._emitLogger("error", "core abort");
}
}
}),
new Promise((_, reject) => {
globalThis.setTimeout(() => {
reject(new Error(`Bridge core init timeout (${initTimeoutMs}ms)`));
}, initTimeoutMs);
})
]);
this._coreVariant = candidate.variant === "wasm64" ? "wasm64" : "wasm32";
if (candidate.variant === "wasm64") {
this._runtimeNotes.push("core_mem64_active");
} else if (candidate.variant === "wasm32") {
this._runtimeNotes.push("core_wasm32_active");
}
break;
} catch (error) {
lastError = error;
if (candidate.variant === "wasm64") {
this._runtimeNotes.push("core_mem64_unavailable");
continue;
}
throw error;
}
}
if (!this._core) {
throw lastError || new Error("Failed to initialize bridge core module");
}
this._applyCoreLogLevel();
return this._core;
}
async _probeBackends() {
try {
const core = await this._ensureCore();
const probeResult = Number(
await core.ccall("llamadart_webgpu_probe", "number", [], [], { async: true })
);
const json = core.ccall("llamadart_webgpu_backends_json", "string", [], []);
let parsed = [];
try {
parsed = JSON.parse(json || "[]");
} catch (_) {
parsed = [];
}
this._backendLabels = Array.isArray(parsed) ? parsed.map((v) => String(v)) : [];
this._gpuActive = probeResult === 1;
} catch (_err) {
this._backendLabels = [];
this._gpuActive = false;
}
return this._gpuActive;
}
async loadModelFromUrl(url, options = {}, resolvedLoadState = null) {
const operationSignal = options?.signal || null;
const abortMessage = "Model load was cancelled.";
throwIfAborted(operationSignal, abortMessage);
this._abortRequested = false;
this._runtimeNotes = [];
this._suppressedWarmupWarningCount = 0;
this._didReportWarmupWarningSuppression = false;
await this._probeBackends();
throwIfAborted(operationSignal, abortMessage);
const core = await this._ensureCore();
throwIfAborted(operationSignal, abortMessage);
const modelUrls = expandModelShardUrls(url);
if (modelUrls.length === 0) {
throw new Error("Model URL is empty.");
}
this._releaseLoadedModel(core);
this._mmProjSourceUrl = null;
const configuredPoolHint = Number(this._threadPoolSizeHint);
this._syncThreadPoolSizeHintFromCore();
const coreSupportsPthreads = this._coreSupportsPthreads();
this._pushRuntimeNote(`core_pthreads:${coreSupportsPthreads ? 1 : 0}`);
if (!coreSupportsPthreads && Number.isFinite(configuredPoolHint) && configuredPoolHint > 1) {
this._runtimeNotes.push("threads_capped_no_pthread");
}
this._nCtx = Number(options.nCtx) > 0 ? Number(options.nCtx) : this._nCtx;
const requestedThreads = Number(options.nThreads);
if (Number.isFinite(requestedThreads) && requestedThreads > 0) {
this._threads = this._capThreadsToPool(requestedThreads);
} else {
this._threads = this._capThreadsToPool(this._resolveAutoThreadCount());
}
const requestedThreadsBatch = Number(options.nThreadsBatch);
if (Number.isFinite(requestedThreadsBatch) && requestedThreadsBatch > 0) {
this._threadsBatch = this._capThreadsToPool(
requestedThreadsBatch,
{ noteTag: "threads_batch_capped_pool" }
);
} else {
this._threadsBatch = this._threads;
}
const requestedGpuLayers = Number(options.nGpuLayers);
if (Number.isFinite(requestedGpuLayers)) {
this._nGpuLayers = Math.trunc(requestedGpuLayers);
}
const isCpuModelMode = this._nGpuLayers === 0;
const requestedBatch = Number(options.nBatch);
this._nBatch = Number.isFinite(requestedBatch) && requestedBatch > 0 ? Math.max(32, Math.trunc(requestedBatch)) : isCpuModelMode ? Math.min(this._nCtx, 512) : 0;
const requestedUbatch = Number(options.nUbatch);
this._nUbatch = Number.isFinite(requestedUbatch) && requestedUbatch > 0 ? Math.max(32, Math.trunc(requestedUbatch)) : isCpuModelMode ? Math.min(this._nBatch || 256, 256) : 0;
if (this._nBatch > 0 && this._nBatch > this._nCtx) {
this._nBatch = this._nCtx;
}
if (this._nUbatch > 0 && this._nBatch > 0 && this._nUbatch > this._nBatch) {
this._nUbatch = this._nBatch;
}
if (resolvedLoadState) {
Object.assign(this, resolvedLoadState);
this._threads = this._capThreadsToPool(this._threads);
this._threadsBatch = this._capThreadsToPool(this._threadsBatch, { noteTag: "threads_batch_capped_pool" });
} else {
this._resolveNativeLoadOptions(options);
}
if (Number.isFinite(this._threadPoolSizeHint) && this._threadPoolSizeHint > 0) {
this._pushRuntimeNote(`thread_pool_size:${this._threadPoolSizeHint}`);
}
if (!isCrossOriginIsolatedRuntime()) {
this._runtimeNotes.push("threads_capped_no_coi");
if (this._threads > 1) {
this._threads = 1;
}
if (this._threadsBatch > 1) {
this._threadsBatch = 1;
}
}
this._pushRuntimeNote(`threads_batch:${this._threadsBatch}`);
if (this._nBatch > 0) {
this._pushRuntimeNote(`n_batch:${this._nBatch}`);
}
if (this._nUbatch > 0) {
this._pushRuntimeNote(`n_ubatch:${this._nUbatch}`);
}
if (this._nSeqMax > 0) {
this._pushRuntimeNote(`n_seq_max:${this._nSeqMax}`);
}
if (!resolvedLoadState && isCpuModelMode && !Number.isFinite(requestedBatch) && !Number.isFinite(requestedUbatch)) {
this._runtimeNotes.push("cpu_batch_tuned_default");
}
if (this._isSafari && this._nGpuLayers > 0) {
const requestedSafariMaxLayers = Number(options.safariMaxGpuLayers);
const safariMaxGpuLayers = Number.isFinite(requestedSafariMaxLayers) ? Math.max(1, Math.trunc(requestedSafariMaxLayers)) : 1;
if (this._nGpuLayers > safariMaxGpuLayers) {
this._nGpuLayers = safariMaxGpuLayers;
this._runtimeNotes.push(`safari_gpu_layers_capped:${safariMaxGpuLayers}`);
}
}
this._loadedModelUrl = cloneModelSource(url);
if (modelUrls.length > 1) {
this._runtimeNotes.push(`model_split_detected:${modelUrls.length}`);
}
let loadedViaRemoteFetch = false;
let remoteFetchReloadUrl = null;
const remoteFetchReloadChunkBytes = this._resolveRemoteFetchChunkBytes(options);
if (modelUrls.length === 1) {
const remoteResult = await this._tryLoadModelFromRemoteFetchBackend(
core,
modelUrls[0],
options
);
loadedViaRemoteFetch = remoteResult.loaded === true;
if (loadedViaRemoteFetch) {
remoteFetchReloadUrl = modelUrls[0];
}
} else {
this._runtimeNotes.push("model_fetch_backend_skipped_split");
}
if (!loadedViaRemoteFetch) {
ensureFsDirectory(core.FS, "/models");
const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null;
const shardLoaded = new Array(modelUrls.length).fill(0);
const shardTotals = new Array(modelUrls.length).fill(0);
const emitAggregateProgress = () => {
if (!progressCallback) {
return;
}
const loaded = sumProgressValues(shardLoaded);
const total = sumProgressValues(shardTotals);
progressCallback({
loaded,
total: total > 0 ? total : loaded
});
};
const modelPaths = [];
let totalModelBytes = 0;
const maxStreamResumeRetries = Number.isFinite(options.streamResumeRetries) ? Math.max(0, Math.trunc(options.streamResumeRetries)) : 8;
const streamChunkTimeoutMs = this._resolveStreamChunkTimeoutMs(
options,
9e4
);
try {
for (let shardIndex = 0; shardIndex < modelUrls.length; shardIndex += 1) {
const shardUrl = modelUrls[shardIndex];
const fileName = basenameFromUrl(shardUrl);
const modelPath = `/models/${fileName}`;
modelPaths.push(modelPath);
let shardBytes = 0;
let resumeOffset = 0;
let resumeAttempt = 0;
let activeShardUrl = shardUrl;
let knownTotalBytes = 0;
while (true) {
throwIfAborted(operationSignal, abortMessage);
const requestHeaders = resumeOffset > 0 ? { Range: `bytes=${resumeOffset}-` } : null;
const response = await this._getCachedModelResponse(activeShardUrl, {
...options,
requireReadableStream: true,
useCache: requestHeaders ? false : options.useCache,
force: requestHeaders ? true : options.force,
requestHeaders
});
if (!response.ok) {
throw new Error(
`Failed to fetch model shard: ${response.status} ${response.statusText}`
);
}
const responseUrl = typeof response.url === "string" && response.url.length > 0 ? response.url : activeShardUrl;
if (responseUrl !== activeShardUrl) {
activeShardUrl = responseUrl;
if (resumeOffset > 0) {
this._runtimeNotes.push("model_stream_resume_redirect");
}
}
if (resumeOffset > 0 && response.status !== 206) {
this._runtimeNotes.push(`model_stream_resume_status:${response.status}`);
throw new Error(
`Range resume not honored for model shard: ${response.status} ${response.statusText}`
);
}
if (!hasReadableResponseStream(response)) {
this._runtimeNotes.push("model_response_nostream");
const declaredBytes = inferResponseTotalBytes(response, 0);
const declaredText = declaredBytes > 0 ? `${declaredBytes} bytes` : "unknown size";
throw new Error(
`Model response did not expose a readable stream (${declaredText}).`
);
}
this._runtimeNotes.push("model_response_stream");
const responseTotal = inferResponseTotalBytes(response, knownTotalBytes);
if (responseTotal > 0) {
knownTotalBytes = responseTotal;
shardTotals[shardIndex] = responseTotal;
}
try {
shardBytes = await writeResponseToFsFileWithProgress(
response,
core.FS,
modelPath,
progressCallback ? (progress) => {
const loaded = Number(progress?.loaded) || 0;
const total = Number(progress?.total) || 0;
shardLoaded[shardIndex] = loaded;
if (total > 0) {
shardTotals[shardIndex] = total;
}
emitAggregateProgress();
} : null,
{
useBigIntPosition: this._coreVariant === "wasm64",
startOffset: resumeOffset,
preservePartialOnError: true,
totalBytes: knownTotalBytes,
chunkTimeoutMs: streamChunkTimeoutMs,
signal: operationSignal,
abortMessage
}
);
break;
} catch (error) {
const text = String(error || "").toLowerCase();
const loadedBytes = Number(
error?.llamadartLoadedBytes
);
if (Number.isFinite(loadedBytes) && loadedBytes >= 0) {
const normalizedLoaded = Math.trunc(loadedBytes);
this._runtimeNotes.push(`model_fs_write_loaded:${normalizedLoaded}`);
if (normalizedLoaded > resumeOffset) {
resumeOffset = normalizedLoaded;
shardLoaded[shardIndex] = normalizedLoaded;
if (knownTotalBytes > 0 && shardTotals[shardIndex] < knownTotalBytes) {
shardTotals[shardIndex] = knownTotalBytes;
}
emitAggregateProgress();
}
}
const shouldRetryResume = isRetryableStreamNetworkError(error) && resumeOffset > 0 && resumeAttempt < maxStreamResumeRetries;
if (shouldRetryResume) {
resumeAttempt += 1;
this._runtimeNotes.push(`model_stream_resume_retry:${resumeAttempt}`);
this._runtimeNotes.push(`model_stream_resume_offset:${resumeOffset}`);
continue;
}
if (text.includes("bigint")) {
this._runtimeNotes.push("model_fs_write_bigint_error");
} else if (text.includes("abort")) {
this._runtimeNotes.push("model_fs_write_abort");
} else if (text.includes("array buffer allocation failed")) {
this._runtimeNotes.push("model_fs_write_arraybuffer_oom");
} else if (isRetryableStreamNetworkError(error)) {
this._runtimeNotes.push("model_fs_write_network_error");
this._runtimeNotes.push("model_fs_write_failed");
} else {
this._runtimeNotes.push("model_fs_write_failed");
}
throw error;
}
}
shardLoaded[shardIndex] = shardBytes;
if (shardTotals[shardIndex] < shardBytes) {
shardTotals[shardIndex] = shardBytes;
}
totalModelBytes += shardBytes;
emitAggregateProgress();
}
this._modelPaths = modelPaths;
this._modelPath = modelPaths[0] || null;
this._modelBytes = totalModelBytes;
let rc = 0;
let nativeLoadStarted = false;
let nativeAbortCleanupDone = false;
try {
throwIfAborted(operationSignal, abortMessage);
nativeLoadStarted = true;
rc = Number(
await core.ccall(
"llamadart_webgpu_load_model",
"number",
[
"string",
"number",
"number",
"number",
"number",
"number",
"number",
...this._nativeLoadOptionTypes()
],
[
this._modelPath,
this._nCtx,
this._threads,
this._threadsBatch,
this._nBatch,
this._nUbatch,
this._nGpuLayers,
...this._nativeLoadOptionValues()
],
{ async: true }
)
);
if (operationSignal?.aborted) {
try {
core.ccall("llamadart_webgpu_shutdown", null, [], []);
nativeAbortCleanupDone = true;
} catch (_) {
}
throw createAbortError(abortMessage);
}
} catch (error) {
if (operationSignal?.aborted && nativeLoadStarted && !nativeAbortCleanupDone) {
try {
core.ccall("llamadart_webgpu_shutdown", null, [], []);
nativeAbortCleanupDone = true;
} catch (_) {
}
error = createAbortError(abortMessage);
}
const text = String(error || "").toLowerCase();
if (text.includes("bigint")) {
this._runtimeNotes.push("model_load_ccall_bigint_error");
} else if (text.includes("abort")) {
this._runtimeNotes.push("model_load_ccall_abort");
} else {
this._runtimeNotes.push("model_load_ccall_failed");
}
throw error;
}
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to load GGUF model", rc));
}
if (modelUrls.length > 1) {
this._runtimeNotes.push(`model_split_loaded:${modelUrls.length}`);
}
} catch (error) {
this._modelBytes = 0;
this._modelPath = null;
this._modelPaths = [];
if (operationSignal?.aborted) {
this._loadedModelUrl = null;
}
for (const modelPath of modelPaths) {
this._deleteFsFile(modelPath);
}
throw error;
}
}
const shouldProbeSafariGpu = this._isSafari && this._nGpuLayers > 0 && options.safariGpuProbe !== false;
if (shouldProbeSafariGpu) {
if (loadedViaRemoteFetch) {
this._runtimeNotes.push("safari_probe_on_fetch_backend");
}
const defaultProbePrompts = [
"user: hi\nassistant:",
"user: say hello in one short sentence\nassistant:"
];
const probePrompts = Array.isArray(options.safariProbePrompts) ? options.safariProbePrompts.map((v) => String(v || "").trim()).filter((v) => v.length > 0) : typeof options.safariProbePrompt === "string" && options.safariProbePrompt.trim().length > 0 ? [options.safariProbePrompt.trim()] : defaultProbePrompts;
const probeTokensRaw = Number(options.safariProbeTokens);
const probeTokens = Number.isFinite(probeTokensRaw) && probeTokensRaw > 0 ? Math.min(Math.trunc(probeTokensRaw), 96) : 48;
const runProbe = async (probePrompt, probeSeed) => {
try {
const probeOutput = await this.createCompletion(probePrompt, {
nPredict: probeTokens,
temp: 0,
topK: 1,
topP: 1,
penalty: 1,
seed: probeSeed
});
return !looksLikeCorruptedGeneration(probeOutput);
} catch (_) {
return false;
}
};
let initialProbePassed = true;
for (let i = 0; i < probePrompts.length; i += 1) {
const ok = await runProbe(probePrompts[i], i + 1);
if (!ok) {
initialProbePassed = false;
break;
}
}
if (!initialProbePassed) {
this._runtimeNotes.push("safari_gpu_probe_failed");
const retryCandidates = [];
if (this._nGpuLayers > 1) {
retryCandidates.push(1);
}
retryCandidates.push(0);
let stabilized = false;
for (const candidateLayers of retryCandidates) {
try {
core.ccall("llamadart_webgpu_shutdown", null, [], []);
} catch (_) {
}
let retryRc = 0;
if (loadedViaRemoteFetch) {
const reloadUrl = remoteFetchReloadUrl || modelUrls[0] || null;
if (!reloadUrl) {
continue;
}
this._runtimeNotes.push(`safari_probe_remote_fetch_retry:${candidateLayers}`);
retryRc = Number(
await core.ccall(
"llamadart_webgpu_load_model_from_url",
"number",
[
"string",
"number",
"number",
"number",
"number",
"number",
"number",
"number",
...this._nativeLoadOptionTypes()
],
[
reloadUrl,
this._nCtx,
this._threads,
this._threadsBatch,
this._nBatch,
this._nUbatch,
candidateLayers,
remoteFetchReloadChunkBytes,
...this._nativeLoadOptionValues()
],
{ async: true }
)
);
} else {
retryRc = Number(
await core.ccall(
"llamadart_webgpu_load_model",
"number",
[
"string",
"number",
"number",
"number",
"number",
"number",
"number",
...this._nativeLoadOptionTypes()
],
[
this._modelPath,
this._nCtx,
this._threads,
this._threadsBatch,
this._nBatch,
this._nUbatch,
candidateLayers,
...this._nativeLoadOptionValues()
],
{ async: true }
)
);
}
if (retryRc !== 0) {
continue;
}
this._nGpuLayers = candidateLayers;
if (candidateLayers === 0) {
this._runtimeNotes.push("safari_fallback_cpu");
stabilized = true;
break;
}
let retryProbePassed = true;
for (let i = 0; i < probePrompts.length; i += 1) {
const ok = await runProbe(probePrompts[i], i + 11);
if (!ok) {
retryProbePassed = false;
break;
}
}
if (retryProbePassed) {
this._runtimeNotes.push(`safari_gpu_layers_capped:${candidateLayers}`);
stabilized = true;
break;
}
}
if (!stabilized) {
throw new Error("Safari GPU probe failed and fallback attempts were unsuccessful.");
}
} else {
this._runtimeNotes.push("safari_gpu_probe_passed");
}
}
try {
const effectiveNctx = Number(core.ccall("llamadart_webgpu_get_context_size", "number", [], []));
if (effectiveNctx > 0) {
this._nCtx = effectiveNctx;
}
} catch (_) {
}
this._mmProjPath = null;
this._mmSupportsVision = false;
this._mmSupportsAudio = false;
this._mediaFileCounter = 0;
this._stateFileCounter = 0;
this._stagedMediaPaths = [];
this._gpuActive = this._gpuActive && this._nGpuLayers > 0;
if (this._releaseModelFiles() > 0) {
this._runtimeNotes.push("model_file_released");
}
this._emitSuppressedWarmupWarningSummaryIfNeeded();
return 1;
}
async loadMultimodalProjector(url) {
if (!this._core || this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
if (typeof url !== "string" || url.length === 0) {
throw new Error("Multimodal projector URL/path is empty.");
}
const core = await this._ensureCore();
ensureFsDirectory(core.FS, "/mmproj");
const fileName = basenameFromUrl(url);
const mmprojPath = `/mmproj/${fileName}`;
const fetchTimeoutMs = this._resolveFetchTimeoutMs({}, 18e4);
const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs({}, 9e4);
let lastError = null;
for (let attempt = 0; attempt < 2; attempt += 1) {
try {
const response = await this._fetchWithTimeout(
url,
{ cache: "no-store" },
fetchTimeoutMs
);
if (!response.ok) {
throw new Error(
`Failed to fetch multimodal projector: ${response.status} ${response.statusText}`
);
}
this._mmProjPath = mmprojPath;
await writeResponseToFsFileWithProgress(
response,
core.FS,
this._mmProjPath,
null,
{
useBigIntPosition: this._coreVariant === "wasm64",
chunkTimeoutMs
}
);
const rc = Number(
await core.ccall(
"llamadart_webgpu_mmproj_load",
"number",
["string"],
[this._mmProjPath],
{ async: true }
)
);
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to load multimodal projector", rc));
}
this._mmSupportsVision = Number(
core.ccall("llamadart_webgpu_mmproj_supports_vision", "number", [], [])
) === 1;
this._mmSupportsAudio = Number(
core.ccall("llamadart_webgpu_mmproj_supports_audio", "number", [], [])
) === 1;
this._mmProjSourceUrl = url;
return 1;
} catch (error) {
lastError = error;
this._mmProjPath = null;
this._mmSupportsVision = false;
this._mmSupportsAudio = false;
this._deleteFsFile(mmprojPath);
const retryable = isRetryableStreamNetworkError(error);
if (!retryable || attempt >= 1) {
throw error;
}
this._runtimeNotes.push(`mmproj_load_retry:${attempt + 1}`);
}
}
if (lastError) {
throw lastError;
}
throw new Error("Failed to load multimodal projector");
}
async unloadMultimodalProjector() {
if (!this._core) {
this._mmProjPath = null;
this._mmSupportsVision = false;
this._mmSupportsAudio = false;
this._mmProjSourceUrl = null;
return;
}
const mmprojPath = this._mmProjPath;
const rc = Number(
this._core.ccall("llamadart_webgpu_mmproj_free", "number", [], [])
);
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to unload multimodal projector", rc));
}
this._deleteFsFile(mmprojPath);
this._mmProjPath = null;
this._mmSupportsVision = false;
this._mmSupportsAudio = false;
this._mmProjSourceUrl = null;
}
// Bytes the core heap can still allocate, or null for a core without the
// probe.
_heapHeadroomBytes(core) {
try {
const value = Number(core.ccall("llamadart_webgpu_heap_headroom_bytes", "number", [], []));
return Number.isFinite(value) && value >= 0 ? value : null;
} catch (_) {
return null;
}
}
// Throws when `bytes` more would not fit in the core heap, naming the
// memory64 core when the wasm32 core's 4 GiB limit is the reason.
_requireHeapHeadroom(core, bytes, what) {
const headroom = this._heapHeadroomBytes(core);
if (headroom == null || !(bytes > headroom)) {
return;
}
const mib = (value, round) => `${round(value / (1024 * 1024))} MiB`;
const detail = this._coreVariant === "wasm32" ? "the wasm32 core addresses at most 4 GiB; memory64 is required: load the bridge with coreModuleUrlMem64 and preferMemory64 in a browser with WebAssembly memory64 support" : "free memory by unloading other models or use a smaller draft model";
throw new Error(
`${what} needs about ${mib(bytes, Math.ceil)} but only ${mib(headroom, Math.floor)} of WebAssembly memory is left; ${detail}.`
);
}
async loadDraftModel(url, options = {}) {
if (!this._core || this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
if (typeof url !== "string" || url.length === 0) {
throw new Error("Draft model URL is empty.");
}
const core = this._core;
const abortMessage = "Draft model load was cancelled.";
throwIfAborted(options.signal || null, abortMessage);
const rc = Number(await core.ccall("llamadart_webgpu_draft_model_free", "number", [], [], { async: true }));
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to release the draft model", rc));
}
this._draftModel = null;
ensureFsDirectory(core.FS, "/draft");
const draftPath = `/draft/${basenameFromUrl(url)}`;
const controller = this._beginTransferAbortController();
const onCallerAbort = () => controller?.abort();
options.signal?.addEventListener("abort", onCallerAbort, { once: true });
const signal = controller?.signal ?? options.signal ?? null;
const targetCacheFields = [this._modelSource, this._modelCacheState, this._modelCacheName];
try {
let response;
try {
response = await this._getCachedModelResponse(url, {
useCache: options.useCache,
force: options.force,
signal,
requireReadableStream: true
});
} finally {
[this._modelSource, this._modelCacheState, this._modelCacheName] = targetCacheFields;
}
if (!response.ok) {
throw new Error(`Failed to fetch draft model: ${response.status} ${response.statusText}`);
}
const totalBytes = inferResponseTotalBytes(response, 0);
if (totalBytes > 0) {
this._requireHeapHeadroom(
core,
this._nGpuLayers === 0 ? totalBytes * 2 : totalBytes,
"The draft model"
);
}
const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null;
await writeResponseToFsFileWithProgress(
response,
core.FS,
draftPath,
progressCallback,
{
useBigIntPosition: this._coreVariant === "wasm64",
totalBytes,
chunkTimeoutMs: this._resolveStreamChunkTimeoutMs({}, 9e4),
signal,
abortMessage
}
);
throwIfAborted(signal, abortMessage);
const loadRc = Number(
await core.ccall(
"llamadart_webgpu_draft_model_load",
"number",
["string"],
[draftPath],
{ async: true }
)
);
if (loadRc !== 0) {
let message = this._coreErrorMessage("Failed to load draft model", loadRc);
if (this._coreVariant === "wasm32" && /memory|alloc/i.test(message)) {
message += "; the wasm32 core addresses at most 4 GiB, so a larger draft model needs the memory64 core";
}
throw new Error(message);
}
const info = JSON.parse(
core.ccall("llamadart_webgpu_draft_model_info_json", "string", [], []) || "{}"
);
this._draftModel = {
url,
options: { useCache: options.useCache, force: false },
info
};
return { ...info };
} finally {
options.signal?.removeEventListener("abort", onCallerAbort);
this._clearTransferAbortController(controller);
this._deleteFsFile(draftPath);
}
}
async unloadDraftModel() {
if (!this._core) {
this._draftModel = null;
return;
}
const rc = Number(
await this._core.ccall("llamadart_webgpu_draft_model_free", "number", [], [], { async: true })
);
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to unload the draft model", rc));
}
this._draftModel = null;
}
// Writes one n-gram cache source to the core filesystem and returns its
// path; the caller deletes it.
async _stageNgramCache(source, name) {
const core = this._core;
ensureFsDirectory(core.FS, "/speculative");
this._speculativeFileCounter += 1;
const path = `/speculative/${name}_${Date.now()}_${this._speculativeFileCounter}.lcs`;
if (typeof source === "string") {
const response = await this._fetchWithTimeout(
source,
{ cache: "no-store" },
this._resolveFetchTimeoutMs({}, 18e4)
);
if (!response.ok) {
throw new Error(`Failed to fetch the ${name} n-gram cache: ${response.status} ${response.statusText}`);
}
core.FS.writeFile(path, new Uint8Array(await response.arrayBuffer()));
} else {
const bytes = source instanceof ArrayBuffer ? new Uint8Array(source) : new Uint8Array(source.buffer, source.byteOffset, source.byteLength);
core.FS.writeFile(path, bytes);
}
return path;
}
// Passes the speculative settings to the core for the next
// begin_generation, which consumes them. Returns the staged cache files.
async _setNextSpeculative(speculative, maxTokens, stagedPaths) {
const core = this._core;
const staticPath = speculative.ngramCacheStatic == null ? "" : await this._stageNgramCache(speculative.ngramCacheStatic, "static");
if (staticPath) {
stagedPaths.push(staticPath);
}
const dynamicPath = speculative.ngramCacheDynamic == null ? "" : await this._stageNgramCache(speculative.ngramCacheDynamic, "dynamic");
if (dynamicPath) {
stagedPaths.push(dynamicPath);
}
const rc = Number(core.ccall(
"llamadart_webgpu_set_next_speculative",
"number",
[
"string",
"number",
"number",
"number",
"number",
"number",
"number",
"number",
"number",
"number",
"number",
"string",
"string",
"number"
],
[
speculative.strategies.join(","),
speculative.draftTokenMax,
speculative.draftTokenMin,
speculative.minProbability,
speculative.draftSplitProbability,
speculative.ngramSizeN,
speculative.ngramSizeM,
speculative.ngramMinHits,
speculative.ngramMatch,
speculative.ngramTokenMin,
speculative.ngramTokenMax,
staticPath,
dynamicPath,
maxTokens
]
));
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to configure speculative decoding", rc));
}
}
supportsVision() {
return this._mmSupportsVision;
}
supportsAudio() {
return this._mmSupportsAudio;
}
getTextToSpeechCapabilities() {
if (!this._core) {
return {
apiVersion: 0,
supported: false,
modelType: 0,
capabilities: 0,
supportsLanguage: false,
supportsSpeakerReference: false,
sampleRate: 0,
channels: 0,
reason: "WebGPU core is not initialized"
};
}
const raw = this._core.ccall(
"llamadart_webgpu_tts_info_json",
"string",
[],
[]
) || "{}";
try {
return JSON.parse(raw);
} catch (_) {
return {
apiVersion: 0,
supported: false,
modelType: 0,
capabilities: 0,
supportsLanguage: false,
supportsSpeakerReference: false,
sampleRate: 0,
channels: 0,
reason: "WebGPU TTS capability response is invalid"
};
}
}
_ensureTextToSpeechDir() {
try {
this._core.FS.mkdir("/tts");
} catch (_) {
}
}
async synthesizeSpeech(options = {}) {
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
if (options.signal?.aborted) {
throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError");
}
const capabilities = this.getTextToSpeechCapabilities();
if (capabilities.supported !== true) {
throw new Error(
capabilities.reason || "Loaded model/projector does not support text-to-speech."
);
}
const text = typeof options.text === "string" ? options.text : "";
if (text.length === 0) {
throw new Error("Text-to-speech input text is empty.");
}
if (this._textToSpeechActive) {
throw new Error("Text-to-speech synthesis is already active.");
}
this._ensureTextToSpeechDir();
const taskId = `${Date.now()}_${++this._mediaFileCounter}`;
const speakerPath = `/tts/speaker_${taskId}.bin`;
const outputPath = `/tts/output_${taskId}.pcm`;
const speakerBytes = toUint8Array(options.speakerAudio);
if (speakerBytes && speakerBytes.length > 0) {
this._core.FS.writeFile(speakerPath, speakerBytes);
}
const promptBatchSize = Number(options.promptBatchSize) > 0 ? Math.max(1, Math.trunc(Number(options.promptBatchSize))) : 512;
const maxFrames = Number(options.maxFrames) > 0 ? Math.max(1, Math.trunc(Number(options.maxFrames))) : 512;
const topK = Number.isFinite(options.topK) ? Math.max(0, Math.trunc(Number(options.topK))) : 40;
const topP = Number.isFinite(options.topP) ? Number(options.topP) : 0.95;
const minP = Number.isFinite(options.minP) ? Number(options.minP) : 0;
const temperature = Number.isFinite(options.temperature) ? Number(options.temperature) : 0.8;
const seed = Number.isInteger(options.seed) ? Number(options.seed) >>> 0 : Math.floor(Math.random() * 4294967295) >>> 0;
const language = typeof options.language === "string" ? options.language : "";
let started = false;
this._textToSpeechActive = true;
this._textToSpeechDone = new Promise((resolve) => {
this._resolveTextToSpeechDone = resolve;
});
try {
this._abortRequested = false;
const startRc = Number(
await this._core.ccall(
"llamadart_webgpu_tts_start",
"number",
["string", "string", "string", "number", "number", "number", "number", "number", "number", "number"],
[
text,
language,
speakerBytes && speakerBytes.length > 0 ? speakerPath : null,
promptBatchSize,
maxFrames,
topK,
topP,
minP,
temperature,
seed
],
{ async: true }
)
);
if (startRc !== 0) {
throw new Error(this._coreErrorMessage("Failed to start text-to-speech synthesis", startRc));
}
started = true;
let progress = {};
for (; ; ) {
if (this._abortRequested || options.signal?.aborted) {
this.cancel();
}
const stepRc = Number(
await this._core.ccall(
"llamadart_webgpu_tts_step",
"number",
[],
[],
{ async: true }
)
);
const rawProgress = this._core.ccall(
"llamadart_webgpu_tts_progress_json",
"string",
[],
[]
) || "{}";
progress = JSON.parse(rawProgress);
options.onProgress?.(progress);
if (stepRc === -6 || progress.state === 4) {
throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError");
}
if (stepRc !== 0) {
throw new Error(this._coreErrorMessage("Text-to-speech synthesis failed", stepRc));
}
if (progress.state === 3) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 0));
}
const outputRc = Number(
this._core.ccall(
"llamadart_webgpu_tts_write_pcm",
"number",
["string"],
[outputPath]
)
);
if (outputRc !== 0) {
throw new Error(this._coreErrorMessage("Failed to read synthesized speech", outputRc));
}
const bytes = this._core.FS.readFile(outputPath);
if (!(bytes instanceof Uint8Array) || bytes.byteLength === 0 || bytes.byteLength % 4 !== 0) {
throw new Error("Synthesized PCM output is empty or malformed.");
}
const copy = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
const pcm = new Float32Array(copy);
return {
pcm,
sampleRate: Number(capabilities.sampleRate) || 0,
channels: Number(capabilities.channels) || 1,
sampleCount: pcm.length,
framesGenerated: Number(progress.framesGenerated) || 0,
truncated: progress.truncated === true
};
} finally {
if (started) {
try {
this._core.ccall("llamadart_webgpu_tts_reset", "number", [], []);
} catch (_) {
}
}
this._deleteFsFile(speakerPath);
this._deleteFsFile(outputPath);
this._textToSpeechActive = false;
this._resolveTextToSpeechDone?.();
this._resolveTextToSpeechDone = null;
this._textToSpeechDone = null;
}
}
getDecisionCapabilities() {
const unsupported = (reason) => ({
apiVersion: DECISION_API_VERSION,
supported: false,
reason
});
const core = this._core;
if (!core) {
return unsupported("WebGPU core is not initialized");
}
if (typeof core._llamadart_webgpu_decision_capabilities_json !== "function") {
return unsupported("This WebGPU core build does not include decision heads.");
}
const coreVersion = Number(
core.ccall("llamadart_webgpu_decision_api_version", "number", [], [])
);
if (coreVersion !== DECISION_API_VERSION) {
return unsupported(
`The WebGPU core implements decision API version ${coreVersion}; this bridge needs version ${DECISION_API_VERSION}.`
);
}
const raw = core.ccall(
"llamadart_webgpu_decision_capabilities_json",
"string",
[],
[]
) || "{}";
try {
return JSON.parse(raw);
} catch (_) {
return unsupported("WebGPU decision capability response is invalid");
}
}
_ensureDecisionDir() {
try {
this._core.FS.mkdir("/decision");
} catch (_) {
}
}
async _stageDecisionHeadFromUrl(url, headPath, onProgress) {
const fetchTimeoutMs = this._resolveFetchTimeoutMs({}, 18e4);
const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs({}, 9e4);
for (let attempt = 0; ; attempt += 1) {
try {
const response = await this._fetchWithTimeout(
url,
{ cache: "no-store" },
fetchTimeoutMs
);
if (!response.ok) {
throw new Error(
`Failed to fetch decision head: ${response.status} ${response.statusText}`
);
}
await writeResponseToFsFileWithProgress(
response,
this._core.FS,
headPath,
typeof onProgress === "function" ? onProgress : null,
{
useBigIntPosition: this._coreVariant === "wasm64",
chunkTimeoutMs
}
);
return;
} catch (error) {
this._unlinkDecisionFile(headPath);
if (!isRetryableStreamNetworkError(error) || attempt >= 1) {
throw error;
}
this._runtimeNotes.push(`decision_head_fetch_retry:${attempt + 1}`);
}
}
}
async loadDecisionHead(source, options = {}) {
if (!this._core || this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
const configJson = options?.configJson ?? null;
if (configJson !== null && typeof configJson !== "string") {
throw new TypeError("Decision head configJson must be a string.");
}
let bytes = null;
if (typeof source === "string") {
if (source.length === 0) {
throw new Error("Decision head URL is empty.");
}
} else {
bytes = bufferSourceBytes(source);
if (!bytes) {
throw new TypeError("Decision head source must be a URL string, an ArrayBuffer or a typed array.");
}
if (bytes.byteLength === 0) {
throw new Error("Decision head bytes are empty.");
}
}
const capabilities = this.getDecisionCapabilities();
if (capabilities.supported !== true) {
throw new Error(
capabilities.reason || "The loaded model does not support decision heads."
);
}
const core = this._core;
this._ensureDecisionDir();
const taskId = `${Date.now()}_${++this._decisionFileCounter}`;
const headPath = `/decision/head_${taskId}.safetensors`;
const configPath = configJson === null ? null : `/decision/config_${taskId}.json`;
try {
let label = "decision head bytes";
if (bytes) {
core.FS.writeFile(headPath, bytes);
} else {
label = basenameFromUrl(source).slice(0, 256);
await this._stageDecisionHeadFromUrl(source, headPath, options?.onProgress);
}
if (configPath !== null) {
core.FS.writeFile(configPath, textEncoder.encode(configJson));
}
const handle = Number(
await core.ccall(
"llamadart_webgpu_decision_load",
"number",
["string", "string", "string"],
[headPath, label, configPath],
{ async: true }
)
);
if (handle <= 0) {
throw new Error(this._coreErrorMessage("Failed to load decision head", handle));
}
const raw = core.ccall(
"llamadart_webgpu_decision_head_info_json",
"string",
["number"],
[handle]
) || "{}";
return JSON.parse(raw);
} finally {
this._unlinkDecisionFile(headPath);
this._unlinkDecisionFile(configPath);
}
}
async runDecision(handle, sequences) {
const nativeHandle = decisionHandleFrom(handle);
const input = encodeDecisionSequences(sequences);
if (!this._core || this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
const core = this._core;
this._ensureDecisionDir();
const taskId = `${Date.now()}_${++this._decisionFileCounter}`;
const inputPath = `/decision/input_${taskId}.bin`;
const outputPath = `/decision/output_${taskId}.bin`;
try {
core.FS.writeFile(inputPath, input);
const count = Number(
await core.ccall(
"llamadart_webgpu_decision_run",
"number",
["number", "string", "string"],
[nativeHandle, inputPath, outputPath],
{ async: true }
)
);
if (count < 0) {
throw new Error(this._coreErrorMessage("Decision run failed", count));
}
return decodeDecisionOutputs(core.FS.readFile(outputPath), count);
} finally {
this._unlinkDecisionFile(inputPath);
this._unlinkDecisionFile(outputPath);
}
}
async freeDecisionHead(handle) {
const nativeHandle = decisionHandleFrom(handle);
if (!this._core) {
return;
}
await this._core.ccall(
"llamadart_webgpu_decision_free",
"number",
["number"],
[nativeHandle],
{ async: true }
);
}
getLoraAdapterCapabilities() {
const unsupported = (reason) => ({
apiVersion: LORA_API_VERSION,
supported: false,
reason
});
const core = this._core;
if (!core) {
return unsupported("WebGPU core is not initialized");
}
if (typeof core._llamadart_webgpu_lora_api_version !== "function") {
return unsupported("This WebGPU core build does not include LoRA adapters.");
}
const coreVersion = Number(
core.ccall("llamadart_webgpu_lora_api_version", "number", [], [])
);
if (coreVersion !== LORA_API_VERSION) {
return unsupported(
`The WebGPU core implements LoRA API version ${coreVersion}; this bridge needs version ${LORA_API_VERSION}.`
);
}
return { apiVersion: LORA_API_VERSION, supported: true };
}
_requireLoraCore() {
if (!this._core || this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
const capabilities = this.getLoraAdapterCapabilities();
if (capabilities.supported !== true) {
throw new Error(capabilities.reason || "This bridge cannot load LoRA adapters.");
}
return this._core;
}
/**
* Downloads a URL adapter into `adapterPath`, through the Cache API unless
* `options.useCache` is false. A cache miss stores the response while it is
* written, so progress starts with the first chunk.
*/
async _stageLoraAdapterFromUrl(url, adapterPath, options) {
const core = this._core;
const signal = options.signal || null;
const progressCallback = typeof options.progressCallback === "function" ? options.progressCallback : null;
const fetchTimeoutMs = this._resolveFetchTimeoutMs(options, 18e4);
const chunkTimeoutMs = this._resolveStreamChunkTimeoutMs(options, 9e4);
const fetchInit = signal ? { signal } : {};
const cacheKey = normalizeAbsoluteUrl(url);
let cache = null;
if (options.useCache !== false && typeof globalThis.caches?.open === "function") {
try {
cache = await globalThis.caches.open(this._resolveCacheName(options));
} catch (_) {
this._runtimeNotes.push("lora_cache_error");
}
}
for (let attempt = 0; ; attempt += 1) {
throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE);
try {
let response = null;
if (cache) {
try {
response = await cache.match(cacheKey) || null;
} catch (_) {
this._runtimeNotes.push("lora_cache_error");
}
}
let stored = null;
if (!response) {
response = await this._fetchWithTimeout(
url,
cache ? fetchInit : { cache: "no-store", ...fetchInit },
fetchTimeoutMs
);
if (!response.ok) {
throw new Error(
`Failed to fetch LoRA adapter: ${response.status} ${response.statusText}`
);
}
if (cache) {
stored = cache.put(cacheKey, response.clone()).then(
() => {
this._runtimeNotes.push("lora_cache_stored");
},
() => {
this._runtimeNotes.push("lora_cache_store_failed");
}
);
}
}
await writeResponseToFsFileWithProgress(
response,
core.FS,
adapterPath,
progressCallback,
{
useBigIntPosition: this._coreVariant === "wasm64",
chunkTimeoutMs,
signal,
abortMessage: LORA_LOAD_ABORT_MESSAGE
}
);
await stored;
return;
} catch (error) {
this._deleteFsFile(adapterPath);
throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE);
if (!isRetryableStreamNetworkError(error) || attempt >= 1) {
throw error;
}
this._runtimeNotes.push(`lora_fetch_retry:${attempt + 1}`);
}
}
}
async loadLoraAdapter(source, options = {}) {
let bytes = null;
if (typeof source === "string") {
if (source.length === 0) {
throw new Error("LoRA adapter URL is empty.");
}
} else {
bytes = bufferSourceBytes(source);
if (!bytes) {
throw new TypeError("LoRA adapter source must be a URL string, an ArrayBuffer or a typed array.");
}
if (bytes.byteLength === 0) {
throw new Error("LoRA adapter bytes are empty.");
}
}
const core = this._requireLoraCore();
const signal = options.signal || null;
throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE);
ensureFsDirectory(core.FS, "/lora");
const adapterPath = `/lora/adapter_${++this._loraFileCounter}.gguf`;
try {
if (bytes) {
core.FS.writeFile(adapterPath, bytes);
} else {
await this._stageLoraAdapterFromUrl(source, adapterPath, options);
}
throwIfAborted(signal, LORA_LOAD_ABORT_MESSAGE);
const handle = Number(
await core.ccall(
"llamadart_webgpu_lora_load",
"number",
["string"],
[adapterPath],
{ async: true }
)
);
if (handle <= 0) {
throw new Error(this._coreErrorMessage("Failed to load LoRA adapter", handle));
}
this._loraAdaptersLoaded = true;
return { handle };
} finally {
this._deleteFsFile(adapterPath);
}
}
async _callLoraCore(name, argTypes, args, failure) {
const core = this._requireLoraCore();
const rc = Number(await core.ccall(name, "number", argTypes, args, { async: true }));
if (rc !== 0) {
throw new Error(this._coreErrorMessage(failure, rc));
}
}
async setLoraAdapter(handle, scale = 1) {
await this._callLoraCore(
"llamadart_webgpu_lora_set",
["number", "number"],
[loraHandleFrom(handle), loraScaleFrom(scale)],
"Failed to apply LoRA adapter"
);
}
async removeLoraAdapter(handle) {
await this._callLoraCore(
"llamadart_webgpu_lora_remove",
["number"],
[loraHandleFrom(handle)],
"Failed to remove LoRA adapter"
);
}
async clearLoraAdapters() {
await this._callLoraCore(
"llamadart_webgpu_lora_clear",
[],
[],
"Failed to clear LoRA adapters"
);
}
_clearStagedMediaFiles() {
if (!this._core || this._stagedMediaPaths.length === 0) {
this._stagedMediaPaths = [];
return;
}
for (const mediaPath of this._stagedMediaPaths) {
try {
this._core.FS.unlink(mediaPath);
} catch (_) {
}
}
this._stagedMediaPaths = [];
}
_clearPendingMedia() {
this._core?.ccall("llamadart_webgpu_media_clear_pending", null, [], []);
this._clearStagedMediaFiles();
}
_persistMediaBytes(bytes, extension = ".bin") {
if (!this._core) {
throw new Error("WebGPU core is not initialized.");
}
try {
this._core.FS.mkdir("/media");
} catch (_) {
}
this._mediaFileCounter += 1;
const suffix = typeof extension === "string" && extension.startsWith(".") ? extension : ".bin";
const mediaPath = `/media/input_${Date.now()}_${this._mediaFileCounter}${suffix}`;
this._core.FS.writeFile(mediaPath, bytes);
this._stagedMediaPaths.push(mediaPath);
return mediaPath;
}
_addMediaFile(mediaPath) {
const rc = Number(
this._core.ccall(
"llamadart_webgpu_media_add_file",
"number",
["string"],
[mediaPath]
)
);
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to add media file", rc));
}
}
_addRawRgbMediaBytes(bytes, width, height) {
const useHeapBuffer = this._core && typeof this._core._malloc === "function" && typeof this._core._free === "function" && this._core.HEAPU8 && typeof this._core.HEAPU8.set === "function";
let rc = 0;
if (useHeapBuffer) {
const ptr = this._core._malloc(bytes.length);
if (!Number.isFinite(ptr) || ptr <= 0) {
throw new Error("Failed to allocate core heap buffer for RGB media bytes");
}
try {
this._core.HEAPU8.set(bytes, ptr);
rc = Number(
this._core.ccall(
"llamadart_webgpu_media_add_rgb",
"number",
["number", "number", "number", "number"],
[width, height, ptr, bytes.length]
)
);
} finally {
this._core._free(ptr);
}
} else {
rc = Number(
this._core.ccall(
"llamadart_webgpu_media_add_rgb",
"number",
["number", "number", "array", "number"],
[width, height, bytes, bytes.length]
)
);
}
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to add raw RGB media bytes", rc));
}
}
_addAudioSamples(samples) {
const sampleBytes = new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength);
const useHeapBuffer = this._core && typeof this._core._malloc === "function" && typeof this._core._free === "function" && this._core.HEAPU8 && typeof this._core.HEAPU8.set === "function";
let rc = 0;
if (useHeapBuffer) {
const ptr = this._core._malloc(sampleBytes.length);
if (!Number.isFinite(ptr) || ptr <= 0) {
throw new Error("Failed to allocate core heap buffer for audio samples");
}
try {
this._core.HEAPU8.set(sampleBytes, ptr);
rc = Number(
this._core.ccall(
"llamadart_webgpu_media_add_audio_f32",
"number",
["number", "number"],
[ptr, samples.length]
)
);
} finally {
this._core._free(ptr);
}
} else {
rc = Number(
this._core.ccall(
"llamadart_webgpu_media_add_audio_f32",
"number",
["array", "number"],
[sampleBytes, samples.length]
)
);
}
if (rc !== 0) {
throw new Error(this._coreErrorMessage("Failed to add audio samples", rc));
}
}
async _fetchMediaBytes(url) {
const response = await this._fetchWithTimeout(
url,
{ cache: "no-store" },
this._resolveFetchTimeoutMs({}, 12e4)
);
if (!response.ok) {
throw new Error(`Failed to fetch media: ${response.status} ${response.statusText}`);
}
return new Uint8Array(await response.arrayBuffer());
}
async _prepareImageBytesForMultimodal(bytes, options = {}) {
const maxPixels = this._resolveMediaImageMaxPixels(options);
const maxEdge = this._resolveMediaImageMaxEdge(options);
if (maxPixels <= 0 && maxEdge <= 0) {
return null;
}
return decodeImageBytesToRgb(bytes, {
maxPixels,
maxEdge
});
}
async _stageMultimodalParts(parts, options = {}) {
this._clearPendingMedia();
const mediaParts = Array.isArray(parts) ? parts : [];
if (mediaParts.length === 0) {
return;
}
if (!this._mmProjPath) {
throw new Error(
"Multimodal input requires a loaded projector. Call loadMultimodalProjector first."
);
}
for (const rawPart of mediaParts) {
const part = rawPart && typeof rawPart === "object" ? rawPart : {};
const type = String(part.type || "").toLowerCase();
if (type === "image") {
const bytes = toUint8Array(part.bytes);
if (bytes && bytes.length > 0) {
const width = Number(part.width);
const height = Number(part.height);
const isRawRgb = Number.isInteger(width) && Number.isInteger(height) && width > 0 && height > 0 && bytes.length === width * height * 3;
if (isRawRgb) {
this._addRawRgbMediaBytes(bytes, width, height);
} else {
const prepared2 = await this._prepareImageBytesForMultimodal(bytes, options);
if (prepared2 && prepared2.bytes && prepared2.bytes.length > 0) {
const mediaPath = this._persistMediaBytes(prepared2.bytes, ".img");
this._addMediaFile(mediaPath);
if (prepared2.resized) {
this._runtimeNotes.push(
`media_image_resized:${prepared2.sourceWidth}x${prepared2.sourceHeight}->${prepared2.width}x${prepared2.height}`
);
}
} else {
const mediaPath = this._persistMediaBytes(bytes, ".img");
this._addMediaFile(mediaPath);
}
}
continue;
}
if (typeof part.url !== "string" || part.url.length === 0) {
throw new Error("Image part must provide bytes or url.");
}
const fetched = await this._fetchMediaBytes(part.url);
const prepared = await this._prepareImageBytesForMultimodal(fetched, options);
if (prepared && prepared.bytes && prepared.bytes.length > 0) {
const mediaPath = this._persistMediaBytes(prepared.bytes, ".img");
this._addMediaFile(mediaPath);
if (prepared.resized) {
this._runtimeNotes.push(
`media_image_resized:${prepared.sourceWidth}x${prepared.sourceHeight}->${prepared.width}x${prepared.height}`
);
}
} else {
const mediaPath = this._persistMediaBytes(fetched, ".img");
this._addMediaFile(mediaPath);
}
continue;
}
if (type === "audio") {
const samples = toFloat32Array(part.samples);
if (samples && samples.length > 0) {
this._addAudioSamples(samples);
continue;
}
const bytes = toUint8Array(part.bytes);
if (bytes && bytes.length > 0) {
const mediaPath2 = this._persistMediaBytes(bytes, ".aud");
this._addMediaFile(mediaPath2);
continue;
}
if (typeof part.url !== "string" || part.url.length === 0) {
throw new Error("Audio part must provide samples, bytes, or url.");
}
const fetched = await this._fetchMediaBytes(part.url);
const mediaPath = this._persistMediaBytes(fetched, ".aud");
this._addMediaFile(mediaPath);
}
}
}
getCompletionCapabilities() {
const core = this._core;
if (!core || typeof core._llamadart_webgpu_completion_capabilities_json !== "function") {
return { ...NO_COMPLETION_CAPABILITIES };
}
return completionCapabilitiesFrom(
core.ccall("llamadart_webgpu_completion_capabilities_json", "string", [], [])
);
}
async createCompletion(prompt, options = {}) {
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
const sampling = resolveCompletionSamplingOptions(options);
requireCompletionCapabilities(sampling, this.getCompletionCapabilities());
this._abortRequested = false;
const startedAt = performance.now();
let firstTokenAt = null;
let nPredict = Number(options.nPredict) > 0 ? Number(options.nPredict) : 256;
const hasMediaParts = Array.isArray(options.parts) && options.parts.length > 0;
if (hasMediaParts) {
const requestedMediaCap = Number(options.mediaMaxPredict);
const mediaCap = Number.isFinite(requestedMediaCap) && requestedMediaCap > 0 ? Math.max(32, Math.trunc(requestedMediaCap)) : 256;
if (nPredict > mediaCap) {
nPredict = mediaCap;
this._runtimeNotes.push(`media_n_predict_capped:${mediaCap}`);
}
}
const temp = Number.isFinite(options.temp) ? Number(options.temp) : 0.8;
const topK = Number.isFinite(options.topK) ? Number(options.topK) : 40;
const topP = Number.isFinite(options.topP) ? Number(options.topP) : 0.95;
const penalty = Number.isFinite(options.penalty) ? Number(options.penalty) : 1.1;
const grammar = typeof options.grammar === "string" && options.grammar.length > 0 ? options.grammar : null;
const seed = Number.isInteger(options.seed) ? Number(options.seed) : Math.floor(Math.random() * 4294967295);
await this._stageMultimodalParts(options.parts, options);
let generationStarted = false;
const speculativePaths = [];
try {
if (sampling.speculative) {
await this._setNextSpeculative(sampling.speculative, nPredict, speculativePaths);
}
const beginRc = Number(
await this._core.ccall(
"llamadart_webgpu_begin_generation",
"number",
[
"string",
"number",
"number",
"number",
"number",
"string",
"number",
"number",
"number",
"number",
"string",
"string",
"string"
],
[
String(prompt),
temp,
topK,
topP,
penalty,
grammar,
seed >>> 0,
sampling.minP,
sampling.presencePenalty,
sampling.thinkingBudget?.maxTokens ?? 0,
sampling.thinkingBudget?.startTag ?? null,
sampling.thinkingBudget?.endTag ?? null,
sampling.thinkingBudget?.forcedMessage ?? null
],
{ async: true }
)
);
if (beginRc === GENERATION_ALREADY_ACTIVE_RC) {
throw new Error(GENERATION_ALREADY_ACTIVE_MESSAGE);
}
if (beginRc !== 0) {
throw new Error(this._coreErrorMessage("Failed to start generation", beginRc));
}
generationStarted = true;
for (const path of speculativePaths.splice(0)) {
this._deleteFsFile(path);
}
let generated = 0;
const shouldEmitCurrentText = options.emitCurrentTextOnToken !== false;
const tokenEventEncoding = typeof options.tokenEventEncoding === "string" ? String(options.tokenEventEncoding || "").toLowerCase() : "bytes";
const emitTokenText = tokenEventEncoding === "text";
const yieldInterval = 4;
let streamed = "";
let emittedStableText = "";
let finishReason = "length";
while (generated < nPredict) {
if (this._abortRequested || options.signal?.aborted) {
finishReason = "cancelled";
break;
}
const stepRc = Number(
await this._core.ccall(
"llamadart_webgpu_next_token",
"number",
[],
[],
{ async: true }
)
);
if (stepRc === 0) {
finishReason = this._abortRequested || options.signal?.aborted ? "cancelled" : "stop";
break;
}
if (stepRc < 0) {
const stepErrorText = this._coreErrorMessage("Generation step failed", stepRc);
if (generated > 0 && this._isContextLimitGenerationError(stepErrorText)) {
this._runtimeNotes.push("generation_stopped_context_limit");
this._emitLogger(
"warn",
"warning: generation reached context/memory limit; returning partial output."
);
break;
}
if (sampling.speculative == null && this._shouldAttemptGenerationRecovery(stepErrorText, options, generated)) {
const recovered = await this._recoverGenerationWithCpuFallback(options);
if (recovered) {
if (generationStarted) {
try {
this._core.ccall("llamadart_webgpu_end_generation", null, [], []);
} catch (_) {
}
generationStarted = false;
}
this._clearPendingMedia();
const retryOptions = {
...options,
_llamadartGenerationRecoveryAttempted: true
};
return await this.createCompletion(prompt, retryOptions);
}
}
throw new Error(stepErrorText);
}
generated += 1;
const fullText = this._core.ccall("llamadart_webgpu_last_output", "string", [], []) || "";
streamed = fullText;
const stableText = trimUnstableUtf8Tail(fullText);
if (!stableText.startsWith(emittedStableText)) {
emittedStableText = "";
}
const deltaText = stableText.slice(emittedStableText.length);
if (deltaText.length === 0) {
continue;
}
emittedStableText = stableText;
firstTokenAt ??= performance.now();
if (typeof options.onToken === "function") {
const piecePayload = emitTokenText ? deltaText : textEncoder.encode(deltaText);
options.onToken(piecePayload, shouldEmitCurrentText ? fullText : null);
}
if (yieldInterval > 0 && generated % yieldInterval === 0) {
await new Promise((resolve) => setTimeout(resolve, 0));
}
}
const text = this._core.ccall("llamadart_webgpu_last_output", "string", [], []) || streamed || "";
const tailText = text.startsWith(emittedStableText) ? text.slice(emittedStableText.length) : "";
if (tailText.length > 0) {
firstTokenAt ??= performance.now();
if (typeof options.onToken === "function") {
const piecePayload = emitTokenText ? tailText : textEncoder.encode(tailText);
options.onToken(piecePayload, shouldEmitCurrentText ? text : null);
}
}
if (typeof options.onUsage === "function") {
const counts = JSON.parse(
this._core.ccall("llamadart_webgpu_last_generation_usage_json", "string", [], []) || "{}"
);
const speculative = counts?.speculative;
options.onUsage({
promptTokens: Number(counts?.promptTokens) || 0,
cachedPromptTokens: Number(counts?.cachedPromptTokens) || 0,
completionTokens: Number(counts?.completionTokens) || 0,
timeToFirstTokenMs: firstTokenAt == null ? null : firstTokenAt - startedAt,
durationMs: performance.now() - startedAt,
finishReason,
...speculative && typeof speculative === "object" ? {
speculative: {
draftTokens: Number(speculative.draftTokens) || 0,
acceptedDraftTokens: Number(speculative.acceptedDraftTokens) || 0,
draftAttempts: Number(speculative.draftAttempts) || 0,
verifyTokens: Number(speculative.verifyTokens) || 0,
replayTokens: Number(speculative.replayTokens) || 0
}
} : {}
});
}
return text;
} finally {
if (generationStarted) {
await this._core.ccall("llamadart_webgpu_end_generation", null, [], [], { async: true });
}
for (const path of speculativePaths) {
this._deleteFsFile(path);
}
this._clearPendingMedia();
}
}
async tokenize(text, _addSpecial = true) {
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
const rc = Number(
await this._core.ccall(
"llamadart_webgpu_tokenize_to_json",
"number",
["string", "number"],
[String(text), _addSpecial ? 1 : 0],
{ async: true }
)
);
if (rc < 0) {
throw new Error(this._coreErrorMessage("Tokenization failed", rc));
}
const raw = this._core.ccall("llamadart_webgpu_last_tokens_json", "string", [], []) || "[]";
const parsed = JSON.parse(raw);
return Array.isArray(parsed) ? parsed.map((v) => Number(v) | 0) : [];
}
_ensureStateDir() {
const core = this._core;
if (!core?.FS) {
throw new Error("Bridge filesystem is not initialized");
}
ensureFsDirectory(core.FS, "/states");
}
_normalizeStateTokens(tokens) {
const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []);
return normalized.map((value) => Number(value) | 0);
}
_nextStateTempPath() {
this._ensureStateDir();
this._stateFileCounter += 1;
return `/states/state_${Date.now()}_${this._stateFileCounter}.bin`;
}
async stateSaveFile(path, tokens = []) {
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
if (typeof path !== "string" || path.length === 0) {
throw new Error("State file path is empty.");
}
const normalized = this._normalizeStateTokens(tokens);
const tokenText = JSON.stringify(normalized);
const rc = Number(
await this._core.ccall(
"llamadart_webgpu_state_save_file",
"number",
["string", "string"],
[path, tokenText],
{ async: true }
)
);
if (rc < 0) {
throw new Error(this._coreErrorMessage("State save failed", rc));
}
return true;
}
async stateLoadFile(path, tokenCapacity = this.getContextSize()) {
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
if (typeof path !== "string" || path.length === 0) {
throw new Error("State file path is empty.");
}
const capacity = Number(tokenCapacity) > 0 ? Math.trunc(Number(tokenCapacity)) : this.getContextSize();
const rc = Number(
await this._core.ccall(
"llamadart_webgpu_state_load_file",
"number",
["string", "number"],
[path, capacity],
{ async: true }
)
);
if (rc < 0) {
throw new Error(this._coreErrorMessage("State load failed", rc));
}
const raw = this._core.ccall("llamadart_webgpu_last_tokens_json", "string", [], []) || "[]";
const parsed = JSON.parse(raw);
const restoredTokens = Array.isArray(parsed) ? parsed.map((v) => Number(v) | 0) : [];
return { tokens: restoredTokens };
}
async stateSaveBytes(tokens = []) {
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
const tempPath = this._nextStateTempPath();
try {
await this.stateSaveFile(tempPath, tokens);
const bytes = this._core.FS.readFile(tempPath);
return bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes || []);
} finally {
this._deleteFsFile(tempPath);
}
}
async stateLoadBytes(bytes, tokenCapacity = this.getContextSize()) {
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
const normalizedBytes = toUint8Array(bytes);
if (!normalizedBytes || normalizedBytes.length === 0) {
throw new Error("State bytes are empty.");
}
const tempPath = this._nextStateTempPath();
try {
this._core.FS.writeFile(tempPath, normalizedBytes);
return await this.stateLoadFile(tempPath, tokenCapacity);
} finally {
this._deleteFsFile(tempPath);
}
}
async detokenize(tokens, _special = false) {
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []);
const tokenText = JSON.stringify(normalized.map((v) => Number(v) | 0));
const rc = Number(
await this._core.ccall(
"llamadart_webgpu_detokenize_from_json",
"number",
["string", "number"],
[tokenText, _special ? 1 : 0],
{ async: true }
)
);
if (rc < 0) {
throw new Error(this._coreErrorMessage("Detokenization failed", rc));
}
return this._core.ccall("llamadart_webgpu_last_detokenized", "string", [], []) || "";
}
_embeddingAssetVersion() {
return embeddingAssetVersion(this._coreVariant === "wasm64" ? this._config.coreModuleUrlMem64 : this._config.coreModuleUrl);
}
async getEmbeddingCapabilities() {
if (this._modelBytes <= 0) return { ...EMPTY_EMBEDDING_CAPABILITIES };
if (typeof this._core?._llamadart_webgpu_embedding_capabilities_json !== "function" || typeof this._core?._llamadart_webgpu_embed_parts_to_json !== "function") {
throw missingEmbeddingCapability(this._embeddingAssetVersion());
}
return embeddingCapabilitiesFrom(
this._core.ccall("llamadart_webgpu_embedding_capabilities_json", "string", [], []),
this._embeddingAssetVersion()
);
}
async _preflightEmbeddingInputs(inputs) {
if (!inputs.some(Array.isArray)) return;
const caps = await this.getEmbeddingCapabilities();
for (const input of inputs) requireEmbeddingModalities(input, caps, !!this._mmProjPath);
}
async _embedParts(parts, options) {
const prompt = parts.map((part) => part.type === "text" ? part.text : EMBEDDING_MEDIA_MARKER).join("");
const promptBytes = textEncoder.encode(prompt);
const media = parts.filter((part) => part.type !== "text");
try {
await this._stageMultimodalParts(media);
if (this._abortRequested) throw createAbortError("Embedding was cancelled.");
const rc = Number(await this._core.ccall(
"llamadart_webgpu_embed_parts_to_json",
"number",
["array", "number", "number"],
[promptBytes, options?.normalize !== false ? 1 : 0, promptBytes.length],
{ async: true }
));
if (this._abortRequested) throw createAbortError("Embedding was cancelled.");
if (rc < 0) throw embeddingCoreError(rc, this._embeddingAssetVersion());
const raw = this._core.ccall("llamadart_webgpu_last_embedding_json", "string", [], []) || "[]";
const parsed = JSON.parse(raw);
return Array.isArray(parsed) ? parsed.map((v) => Number.isFinite(Number(v)) ? Number(v) : 0) : [];
} catch (error) {
if (error instanceof WebAssembly.RuntimeError) throw error;
if (error instanceof EmbeddingError || error?.name === "AbortError") throw error;
throw new EmbeddingError("EMBEDDING_FAILED", "Embedding media preparation or execution failed.");
} finally {
this._clearPendingMedia();
}
}
async embed(text, options = {}) {
if (Array.isArray(text)) {
const parts = snapshotEmbedInput(text);
this._abortRequested = false;
await this._preflightEmbeddingInputs([parts]);
if (this._abortRequested) throw createAbortError("Embedding was cancelled.");
return this._embedParts(parts, options);
}
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
const normalize = options?.normalize !== false;
const rc = Number(
await this._core.ccall(
"llamadart_webgpu_embed_to_json",
"number",
["string", "number"],
[String(text), normalize ? 1 : 0],
{ async: true }
)
);
if (rc < 0) {
throw new Error(this._coreErrorMessage("Embedding generation failed", rc));
}
const raw = this._core.ccall("llamadart_webgpu_last_embedding_json", "string", [], []) || "[]";
const parsed = JSON.parse(raw);
return Array.isArray(parsed) ? parsed.map((v) => {
const numeric = Number(v);
return Number.isFinite(numeric) ? numeric : 0;
}) : [];
}
async embedBatch(texts, options = {}) {
const normalized = Array.isArray(texts) ? texts : Array.from(texts || []);
if (normalized.length === 0) {
return [];
}
const inputs = normalized.map((input) => typeof input === "string" ? input : snapshotEmbedInput(input));
this._abortRequested = false;
await this._preflightEmbeddingInputs(inputs);
const normalize = options?.normalize !== false;
const vectors = [];
for (const text of inputs) {
if (this._abortRequested) throw createAbortError("Batch embedding was cancelled.");
vectors.push(await (typeof text === "string" ? this.embed(text, { normalize }) : this._embedParts(text, { normalize })));
}
return vectors;
}
async scoreNextToken(prompt, options = {}) {
if (this._modelBytes <= 0) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
if (typeof prompt !== "string") {
throw new TypeError("Next-token scoring prompt must be a string.");
}
const candidates = nextTokenCandidateIds(options?.candidates);
const topK = options?.topK ?? 0;
if (!isInt32(topK)) {
throw new TypeError(`Next-token topK is ${String(topK)}; expected a 32-bit integer.`);
}
const reusePromptPrefix = options?.reusePromptPrefix !== false;
const rc = Number(
await this._core.ccall(
"llamadart_webgpu_score_next_token_to_json",
"number",
["string", "string", "number", "number"],
[prompt, JSON.stringify(candidates), topK, reusePromptPrefix ? 1 : 0],
{ async: true }
)
);
if (rc < 0) {
throw new Error(this._coreErrorMessage("Next-token scoring failed", rc));
}
const raw = this._core.ccall("llamadart_webgpu_last_next_token_scores_json", "string", [], []) || "{}";
const parsed = JSON.parse(raw);
return {
candidates: scoredTokensFrom(parsed?.candidates),
top: scoredTokensFrom(parsed?.top),
promptTokens: Number(parsed?.promptTokens) || 0
};
}
getModelMetadata() {
let modelMetadata = {};
try {
const raw = this._core?.ccall("llamadart_webgpu_model_meta_json", "string", [], []);
if (raw) {
const parsed = JSON.parse(raw);
if (parsed && typeof parsed === "object") {
modelMetadata = parsed;
}
}
} catch (_) {
}
return {
...modelMetadata,
"llamadart.webgpu.prototype": "1",
"llamadart.webgpu.backends": this._backendLabels.join(","),
"llamadart.webgpu.model_bytes": String(this._modelBytes),
"llamadart.webgpu.n_threads": String(this._threads),
"llamadart.webgpu.n_threads_batch": String(this._threadsBatch),
"llamadart.webgpu.n_batch": this._nBatch > 0 ? String(this._nBatch) : "",
"llamadart.webgpu.n_ubatch": this._nUbatch > 0 ? String(this._nUbatch) : "",
"llamadart.webgpu.n_seq_max": this._nSeqMax > 0 ? String(this._nSeqMax) : "",
"llamadart.webgpu.flash_attention": String(this._flashAttention),
"llamadart.webgpu.cache_type_k": String(this._cacheTypeK),
"llamadart.webgpu.cache_type_v": String(this._cacheTypeV),
"llamadart.webgpu.kv_unified": this._kvUnified >= 0 ? String(this._kvUnified) : "",
"llamadart.webgpu.rope_freq_base": this._ropeFrequencyBase > 0 ? String(this._ropeFrequencyBase) : "",
"llamadart.webgpu.rope_freq_scale": this._ropeFrequencyScale > 0 ? String(this._ropeFrequencyScale) : "",
"llamadart.webgpu.split_mode": this._splitMode >= 0 ? String(this._splitMode) : "",
"llamadart.webgpu.main_gpu": this._mainGpu >= 0 ? String(this._mainGpu) : "",
"llamadart.webgpu.thread_pool_size": Number.isFinite(this._threadPoolSizeHint) && this._threadPoolSizeHint > 0 ? String(this._threadPoolSizeHint) : "",
"llamadart.webgpu.n_gpu_layers": String(this._nGpuLayers),
"llamadart.webgpu.core_variant": this._coreVariant,
"llamadart.webgpu.model_source": this._modelSource,
"llamadart.webgpu.model_cache_state": this._modelCacheState,
"llamadart.webgpu.model_cache_name": this._modelCacheName,
"llamadart.webgpu.runtime_notes": this._runtimeNotes.join(";"),
"llamadart.webgpu.mmproj_loaded": this._mmProjPath ? "1" : "0",
"llamadart.webgpu.supports_vision": this._mmSupportsVision ? "1" : "0",
"llamadart.webgpu.supports_audio": this._mmSupportsAudio ? "1" : "0"
};
}
getContextSize() {
try {
const nctx = Number(this._core?.ccall("llamadart_webgpu_get_context_size", "number", [], []));
if (nctx > 0) {
return nctx;
}
} catch (_) {
}
return this._nCtx;
}
isGpuActive() {
return this._gpuActive;
}
getBackendName() {
if (this._nGpuLayers === 0) {
return "WASM (Prototype bridge)";
}
if (this._backendLabels.length > 0) {
return this._backendLabels.join(", ");
}
return this._gpuActive ? "WebGPU (Prototype bridge)" : "WASM (Prototype bridge)";
}
setLogLevel(level) {
if (Number.isFinite(level)) {
this._logLevel = Math.max(0, Math.min(4, Math.trunc(level)));
}
this._applyCoreLogLevel();
}
cancel() {
this._abortRequested = true;
if (this._activeTransferAbortController) {
try {
this._activeTransferAbortController.abort();
} catch (_) {
}
this._activeTransferAbortController = null;
}
try {
this._core?.ccall("llamadart_webgpu_request_cancel", null, [], []);
} catch (_) {
}
}
async dispose() {
if (this._textToSpeechActive && this._textToSpeechDone) {
this.cancel();
await this._textToSpeechDone;
}
if (this._activeTransferAbortController) {
try {
this._activeTransferAbortController.abort();
} catch (_) {
}
this._activeTransferAbortController = null;
}
if (this._core) {
this._clearPendingMedia();
const mmprojPath = this._mmProjPath;
const mmprojFreeRc = Number(
this._core.ccall("llamadart_webgpu_mmproj_free", "number", [], [])
);
this._core.ccall("llamadart_webgpu_shutdown", null, [], []);
this._deleteFsFile(mmprojPath);
if (mmprojFreeRc !== 0) {
this._emitLogger(
"warn",
this._coreErrorMessage("Failed to unload multimodal projector during dispose", mmprojFreeRc)
);
}
}
this._modelPath = null;
this._modelPaths = [];
this._modelBytes = 0;
this._modelSource = "network";
this._modelCacheState = "disabled";
this._loadedModelUrl = null;
this._mmProjPath = null;
this._mmProjSourceUrl = null;
this._mmSupportsVision = false;
this._mmSupportsAudio = false;
this._loraAdaptersLoaded = false;
this._abortRequested = false;
this._textToSpeechActive = false;
this._textToSpeechDone = null;
this._resolveTextToSpeechDone = null;
this._activeTransferAbortController = null;
this._suppressedWarmupWarningCount = 0;
this._didReportWarmupWarningSuppression = false;
}
async applyChatTemplate(messages, addAssistant = true, _customTemplate = null) {
return buildPromptFromMessages(messages, addAssistant);
}
};
// js/src/worker_protocol.ts
function serializeWorkerError(error) {
if (!error) {
return "Unknown worker error";
}
if (typeof error === "string") {
return error;
}
if (typeof error.message === "string" && error.message.length > 0) {
return error.message;
}
try {
return JSON.stringify(error);
} catch (_) {
return String(error);
}
}
var bridgeWorkerModeParam = "__llamadartBridgeWorker";
function emptyBridgeState() {
return {
metadata: {},
contextSize: 0,
gpuActive: false,
backendName: "WASM (Prototype bridge)",
supportsVision: false,
supportsAudio: false
};
}
function snapshotBridgeState(target) {
return {
metadata: target.getModelMetadata(),
contextSize: target.getContextSize(),
gpuActive: target.isGpuActive(),
backendName: target.getBackendName(),
supportsVision: target.supportsVision(),
supportsAudio: target.supportsAudio()
};
}
// js/src/worker_proxy.ts
function createBridgeWorkerSource(moduleUrl) {
return `import * as workerModule from ${JSON.stringify(moduleUrl)};
if (workerModule && typeof workerModule.enableBridgeWorkerHost === 'function') { workerModule.enableBridgeWorkerHost(); }
`;
}
function resolveWorkerEntryUrl(moduleUrl) {
if (typeof moduleUrl !== "string" || moduleUrl.length === 0) {
return null;
}
try {
const base = typeof window !== "undefined" && window.location?.href ? window.location.href : void 0;
const url = new URL(moduleUrl, base);
const path = url.pathname || "";
const usesDedicatedWorkerEntry = path.endsWith("_worker.js");
if (!usesDedicatedWorkerEntry) {
url.searchParams.set(bridgeWorkerModeParam, "1");
}
return url.toString();
} catch (_) {
return null;
}
}
function deriveBridgeModuleUrlFromWorkerEntry(moduleUrl) {
if (typeof moduleUrl !== "string" || moduleUrl.length === 0) {
return null;
}
try {
const base = typeof window !== "undefined" && window.location?.href ? window.location.href : void 0;
const url = new URL(moduleUrl, base);
const path = url.pathname || "";
if (!path.endsWith("_worker.js")) {
return null;
}
url.pathname = path.replace(/_worker\.js$/, ".js");
url.searchParams.delete(bridgeWorkerModeParam);
return url.toString();
} catch (_) {
return null;
}
}
var BridgeWorkerProxy = class {
constructor({ moduleUrl, config }) {
this._config = config && typeof config === "object" ? config : {};
this._nextId = 1;
this._pending = /* @__PURE__ */ new Map();
this._workerBlobUrl = null;
let workerInitError = null;
const moduleCandidates = [moduleUrl];
const bridgeModuleFallback = deriveBridgeModuleUrlFromWorkerEntry(moduleUrl);
if (bridgeModuleFallback && bridgeModuleFallback !== moduleUrl) {
moduleCandidates.push(bridgeModuleFallback);
}
for (const candidate of moduleCandidates) {
if (this._worker) {
break;
}
const directWorkerUrl = resolveWorkerEntryUrl(candidate);
if (!directWorkerUrl) {
continue;
}
try {
this._worker = new Worker(directWorkerUrl, { type: "module" });
} catch (error) {
workerInitError = error;
}
}
for (const candidate of moduleCandidates) {
if (this._worker) {
break;
}
const source = createBridgeWorkerSource(candidate);
this._workerBlobUrl = URL.createObjectURL(
new Blob([source], { type: "text/javascript" })
);
try {
this._worker = new Worker(this._workerBlobUrl, { type: "module" });
} catch (error) {
workerInitError = error;
URL.revokeObjectURL(this._workerBlobUrl);
this._workerBlobUrl = null;
}
}
if (!this._worker) {
throw workerInitError || new Error("Failed to initialize bridge worker");
}
this._ready = new Promise((resolve, reject) => {
this._readyResolve = resolve;
this._readyReject = reject;
});
this._readyTimeoutHandle = null;
this._armReadyTimeout();
this._worker.onmessage = (event) => {
const message = event.data || {};
const type = message.type;
if (type === "ready") {
this._clearReadyTimeout();
this._readyResolve?.();
return;
}
const id = Number(message.id || 0);
const pending = this._pending.get(id);
if (!pending) {
return;
}
if (type === "event") {
pending.onEvent?.(message);
return;
}
this._pending.delete(id);
if (type === "error") {
const errorMessage = String(message.message || "Worker request failed");
const workerError = message.embeddingErrorCode ? new EmbeddingError(message.embeddingErrorCode, errorMessage) : new Error(errorMessage);
if (message.state && typeof message.state === "object") {
workerError.state = message.state;
}
pending.reject(workerError);
return;
}
pending.resolve(message);
};
this._worker.onerror = (event) => {
const message = event?.message || "Bridge worker crashed";
const error = new Error(String(message));
error.llamadartWorkerCrash = true;
this._clearReadyTimeout();
this._readyReject?.(error);
for (const pending of this._pending.values()) {
pending.reject(error);
}
this._pending.clear();
};
this._worker.postMessage({ type: "init", config });
}
async call(method, args, onEvent, transferList = [], operationMeta = null) {
await this._ready;
const id = this._nextId++;
const timeoutMs = this._resolveRequestTimeoutMs(method, args);
const transfers = Array.isArray(transferList) ? transferList.filter((item) => item != null) : [];
return new Promise((resolve, reject) => {
let timeoutHandle = null;
const clearTimer = () => {
if (timeoutHandle != null) {
globalThis.clearTimeout(timeoutHandle);
timeoutHandle = null;
}
};
const armTimer = () => {
clearTimer();
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
return;
}
timeoutHandle = globalThis.setTimeout(() => {
this._pending.delete(id);
reject(new Error(`Worker request timeout (${method}, ${timeoutMs}ms)`));
}, timeoutMs);
};
armTimer();
this._pending.set(id, {
resolve: (value) => {
clearTimer();
resolve(value);
},
reject: (error) => {
clearTimer();
reject(error);
},
onEvent: (event) => {
armTimer();
onEvent?.(event);
}
});
const message = { type: "call", id, method, args };
if (operationMeta && typeof operationMeta === "object") {
Object.assign(message, operationMeta);
}
this._worker.postMessage(message, transfers);
});
}
_resolveWorkerReadyTimeoutMs() {
const configured = Number(this._config.workerInitTimeoutMs);
if (!Number.isFinite(configured) || configured <= 0) {
return 2e4;
}
return Math.max(3e3, Math.min(12e4, Math.trunc(configured)));
}
_clearReadyTimeout() {
if (this._readyTimeoutHandle != null) {
globalThis.clearTimeout(this._readyTimeoutHandle);
this._readyTimeoutHandle = null;
}
}
_armReadyTimeout() {
this._clearReadyTimeout();
const timeoutMs = this._resolveWorkerReadyTimeoutMs();
this._readyTimeoutHandle = globalThis.setTimeout(() => {
this._readyTimeoutHandle = null;
const timeoutError = new Error(`Bridge worker init timeout (${timeoutMs}ms)`);
this._readyReject?.(timeoutError);
try {
this._worker?.terminate();
} catch (_) {
}
}, timeoutMs);
}
_resolveRequestTimeoutMs(method, args = []) {
const explicitGlobal = Number(this._config.workerRequestTimeoutMs);
const clamp = (value, fallback) => {
if (!Number.isFinite(value) || value <= 0) {
return fallback;
}
return Math.max(5e3, Math.min(36e5, Math.trunc(value)));
};
if (method === "loadModelFromUrl" || method === "loadDraftModel") {
return clamp(Number(this._config.workerModelLoadTimeoutMs), clamp(explicitGlobal, 3 * 60 * 1e3));
}
if (method === "loadMultimodalProjector") {
return clamp(Number(this._config.workerMmprojLoadTimeoutMs), clamp(explicitGlobal, 8 * 60 * 1e3));
}
if (method === "createCompletion") {
return clamp(Number(this._config.workerCompletionTimeoutMs), clamp(explicitGlobal, 6 * 60 * 1e3));
}
if (method === "synthesizeSpeech") {
return clamp(Number(this._config.workerTextToSpeechTimeoutMs), clamp(explicitGlobal, 20 * 60 * 1e3));
}
if (method === "loadDecisionHead" || method === "loadLoraAdapter") {
return clamp(explicitGlobal, 10 * 60 * 1e3);
}
if (method === "runDecision") {
const sequences = Array.isArray(args?.[1]) ? args[1].length : 0;
return clamp(explicitGlobal, 10 * 60 * 1e3 + sequences * DECISION_WORKER_TIMEOUT_PER_SEQUENCE_MS);
}
return clamp(explicitGlobal, 12e4);
}
async dispose() {
this._clearReadyTimeout();
let didTimeout = false;
try {
await Promise.race([
this.call("dispose", []),
new Promise((resolve) => {
globalThis.setTimeout(() => {
didTimeout = true;
resolve(null);
}, 800);
})
]);
} catch (_) {
}
if (didTimeout) {
}
for (const pending of this._pending.values()) {
pending.reject(new Error("Bridge worker disposed"));
}
this._pending.clear();
this._worker.terminate();
if (this._workerBlobUrl) {
URL.revokeObjectURL(this._workerBlobUrl);
this._workerBlobUrl = null;
}
}
};
// js/src/bridge.ts
var LlamaWebGpuBridge = class {
static supportsSafariAdaptiveGpu = LlamaWebGpuBridgeRuntime.supportsSafariAdaptiveGpu === true;
static supportsCompletionUsage = true;
constructor(config = {}) {
this._config = config;
this._runtime = null;
this._workerProxy = null;
this._workerGeneration = 0;
this._workerDisposePromise = null;
this._retiringWorkerDisposals = /* @__PURE__ */ new Set();
this._retiredWorkerProxies = /* @__PURE__ */ new WeakSet();
this._workerFallbackReason = null;
this._metadata = {};
this._contextSize = 0;
this._gpuActive = false;
this._backendName = "WASM (Prototype bridge)";
this._supportsVision = false;
this._supportsAudio = false;
this._loadedModelUrl = null;
this._loadedModelOptions = null;
this._loadedMmProjUrl = null;
this._loadedDraftModel = null;
this._multimodalWorkerCpuMode = false;
this._workerModelMissing = false;
this._bridgeWarnRecent = /* @__PURE__ */ new Map();
this._operationQueueTail = null;
this._activeOperation = null;
this._nextOperationId = 0;
this._lifecycleState = "open";
this._shadowStateTransactionDepth = 0;
this._deferredShadowState = null;
this._disposed = false;
this._disposePromise = null;
this._disposalWaiters = /* @__PURE__ */ new Set();
this._decisionHeads = /* @__PURE__ */ new Map();
this._nextDecisionHandle = 1;
this._loraAdapters = /* @__PURE__ */ new Map();
this._activeLoraScales = /* @__PURE__ */ new Map();
this._nextLoraHandle = 1;
if (this._shouldUseWorker()) {
try {
this._workerProxy = this._createWorkerProxy();
} catch (error) {
this._disableWorkerFallback(error);
}
}
if (this._workerProxy) {
this._workerGeneration = 1;
}
if (!this._workerProxy) {
this._runtime = this._createRuntime();
this._captureDirectRuntimeState();
}
}
_createRuntime() {
return new LlamaWebGpuBridgeRuntime({
...this._config,
disableWorker: true
});
}
_operationSignal(operation = this._activeOperation) {
return operation?.abortController?.signal || null;
}
_recordWorkerGeneration(operation = this._activeOperation, proxy = this._workerProxy) {
if (!operation) {
return;
}
const generation = this._workerGeneration || 0;
operation.workerGeneration = generation;
operation.workerGenerations?.add(generation);
if (proxy) {
operation.workerProxy = proxy;
operation.currentWorkerProxy = proxy;
operation.workerProxies?.add(proxy);
}
}
_advanceWorkerGeneration() {
this._workerGeneration = (this._workerGeneration || 0) + 1;
this._recordWorkerGeneration();
return this._workerGeneration;
}
_isOperationCurrent(operation) {
return !!(operation && this._activeOperation === operation && this._lifecycleState !== "disposed" && operation.state !== "cancelling" && operation.state !== "cancelled" && operation.state !== "disposed");
}
_isOperationCallbackCurrent(operation, proxy) {
return this._isOperationCurrent(operation) && proxy === this._workerProxy;
}
_sendWorkerCancel(proxy) {
if (!proxy) {
return;
}
if (typeof proxy.call !== "function") {
if (proxy === this._workerProxy && typeof this._callWorker === "function") {
try {
Promise.resolve(this._callWorker("cancel", [])).catch(() => {
});
} catch (_) {
}
}
return;
}
try {
Promise.resolve(proxy.call("cancel", [], null)).catch(() => {
});
} catch (_) {
}
}
_cancelOperation(operation, reason = "user") {
if (!operation) {
return;
}
if (reason === "worker-timeout") {
if (operation.cancelRequested || operation.workerTimeoutCancellationSent) {
return;
}
operation.workerTimeoutCancellationSent = true;
operation.cancelReason = reason;
operation.state = "recovering";
const proxies2 = new Set(operation.workerProxies || []);
if (operation.workerProxy) {
proxies2.add(operation.workerProxy);
}
for (const proxy of proxies2) {
this._sendWorkerCancel(proxy);
}
return;
}
if (operation.cancelRequested) {
return;
}
operation.cancelRequested = true;
operation.cancelReason = reason;
operation.state = "cancelling";
try {
operation.abortController?.abort();
} catch (_) {
}
const proxies = new Set(operation.workerProxies || []);
if (operation.workerProxy) {
proxies.add(operation.workerProxy);
}
for (const proxy of proxies) {
this._sendWorkerCancel(proxy);
}
const runtimes = new Set(operation.runtimes || []);
if (operation.runtime) {
runtimes.add(operation.runtime);
}
for (const runtime of runtimes) {
try {
runtime?.cancel?.();
} catch (_) {
}
}
}
_isOperationCancellation(error, operation = this._activeOperation) {
if (operation?.callerSignal?.aborted) {
return true;
}
if (operation?.cancelRequested && operation.cancelReason !== "worker-timeout") {
return true;
}
return error?.name === "AbortError" && !this._isWorkerTimeoutError(error);
}
_throwIfOperationCancelled(error, message = "Bridge operation was cancelled.", operation = this._activeOperation) {
if (this._isOperationCancellation(error, operation)) {
throw createAbortError(message);
}
}
/**
* Guards the success path: a worker that ignored cancellation and settled
* with a full result must still reject once the caller's signal was aborted,
* and must not commit that result. Only the caller's signal counts here.
* A bare `cancel()` stays the documented best-effort control signal, so a
* result that still arrived is delivered rather than discarded.
*/
_throwIfCallerCancelled(message = "Bridge operation was cancelled.", operation = this._activeOperation) {
if (operation?.callerSignal?.aborted) {
throw createAbortError(message);
}
}
_retireWorkerProxy(proxy) {
if (!proxy || typeof proxy !== "object") {
return Promise.resolve();
}
if (this._retiredWorkerProxies?.has(proxy)) {
return Promise.resolve();
}
this._retiredWorkerProxies?.add(proxy);
const operation = this._activeOperation;
operation?.workerProxies?.add(proxy);
let disposal;
try {
disposal = Promise.resolve(proxy.dispose?.()).catch(() => {
});
} catch (_) {
disposal = Promise.resolve();
}
this._retiringWorkerDisposals ??= /* @__PURE__ */ new Set();
this._retiringWorkerDisposals.add(disposal);
this._workerDisposePromise = disposal;
operation?.retirements?.add(disposal);
disposal.then(
() => {
this._retiringWorkerDisposals.delete(disposal);
if (this._workerDisposePromise === disposal && this._retiringWorkerDisposals.size === 0) {
this._workerDisposePromise = null;
}
},
() => {
this._retiringWorkerDisposals.delete(disposal);
if (this._workerDisposePromise === disposal && this._retiringWorkerDisposals.size === 0) {
this._workerDisposePromise = null;
}
}
);
return disposal;
}
async _waitForWorkerDisposal() {
this._retiringWorkerDisposals ??= /* @__PURE__ */ new Set();
while (this._retiringWorkerDisposals.size > 0) {
const pending = [...this._retiringWorkerDisposals];
await Promise.all(pending);
}
this._workerDisposePromise = null;
}
/**
* llama.cpp exposes one model/context per bridge instance and publishes
* results through process-global C buffers (`last_output`, `last_tokens_json`,
* `last_detokenized`, `last_embedding_json`, ...). Every async call that reads
* or mutates that runtime therefore has to own it exclusively for its whole
* duration: a second caller interleaving between the work ccall and the buffer
* read would otherwise observe the other caller's result. This FIFO queue is
* the single-writer lock, shared by the worker and direct-runtime paths.
*
* A caller whose signal is already aborted, or that aborts while still queued,
* never runs and never dispatches; its slot is skipped in turn without
* touching the unrelated operation that currently owns the runtime. Once an
* operation has started, its signal is routed to `_cancelOperation` so the
* cancel reaches the worker generation and runtime that operation captured
* rather than whatever topology happens to be current.
*/
_runExclusive(run, options = {}) {
const signal = options.signal || null;
const abortMessage = options.abortMessage || "Bridge operation was cancelled.";
const allowDisposed = options.allowDisposed === true;
const abortError = () => createAbortError(abortMessage);
const lifecycleState = this._lifecycleState || (this._disposed ? "disposed" : "open");
if (!allowDisposed && (this._disposed || lifecycleState !== "open")) {
return Promise.reject(new Error(BRIDGE_DISPOSED_MESSAGE));
}
if (signal?.aborted) {
return Promise.reject(abortError());
}
const operation = {
id: ++this._nextOperationId,
kind: options.kind || "bridge-operation",
state: "queued",
callerSignal: signal,
abortController: typeof AbortController === "function" ? new AbortController() : null,
cancelRequested: false,
cancelReason: null,
workerTimeoutCancellationSent: false,
workerGeneration: this._workerGeneration || 0,
workerGenerations: /* @__PURE__ */ new Set([this._workerGeneration || 0]),
workerProxy: this._workerProxy || null,
runtime: this._runtime || null,
workerProxies: new Set(this._workerProxy ? [this._workerProxy] : []),
runtimes: new Set(this._runtime ? [this._runtime] : []),
modelSource: null,
modelOptions: null,
projectorSource: null,
recoveryAttempted: false,
generationStarted: false,
retirements: /* @__PURE__ */ new Set(),
deferredShadowState: null
};
const predecessor = this._operationQueueTail;
let releaseSuccessor = () => {
};
const queueSlot = new Promise((resolve) => {
releaseSuccessor = () => resolve(void 0);
});
this._operationQueueTail = queueSlot;
let started = false;
let removeAbortListener = null;
let removeDisposalWaiter = null;
const detachWatchers = () => {
removeAbortListener?.();
removeDisposalWaiter?.();
removeAbortListener = null;
removeDisposalWaiter = null;
};
const watchesAbort = signal != null && typeof signal.addEventListener === "function";
const watchesDisposal = !allowDisposed;
const ordered = (async () => {
if (predecessor) {
await predecessor;
}
if (signal?.aborted) {
operation.state = "skipped";
throw abortError();
}
if (!allowDisposed && (this._disposed || this._lifecycleState === "disposing")) {
operation.state = "disposed";
throw new Error(BRIDGE_DISPOSED_MESSAGE);
}
started = true;
removeDisposalWaiter?.();
removeDisposalWaiter = null;
operation.state = "starting";
operation.workerProxy = this._workerProxy || null;
operation.workerGeneration = this._workerGeneration || 0;
operation.workerGenerations.add(operation.workerGeneration);
operation.runtime = this._runtime || null;
if (operation.workerProxy) {
operation.workerProxies.add(operation.workerProxy);
operation.currentWorkerProxy = operation.workerProxy;
}
if (operation.runtime) {
operation.runtimes.add(operation.runtime);
}
this._activeOperation = operation;
operation.state = "running";
let operationFailed = false;
try {
return await run(operation);
} catch (error) {
operationFailed = true;
throw error;
} finally {
let terminalState = operation.cancelRequested ? "cancelled" : operationFailed ? "failed" : "completed";
operation.state = terminalState === "completed" ? "settling" : terminalState;
try {
this._captureDirectRuntimeState();
} catch (error) {
terminalState = "failed";
throw error;
} finally {
if (this._activeOperation === operation) {
this._activeOperation = null;
}
operation.state = terminalState;
}
}
})();
const release = () => {
if (this._operationQueueTail === queueSlot) {
this._operationQueueTail = null;
}
releaseSuccessor();
};
ordered.then(release, release);
if (!watchesAbort && !watchesDisposal) {
return ordered;
}
const abandonedWhileQueued = new Promise((_, reject) => {
if (watchesAbort) {
const onAbort = () => {
if (!started) {
operation.state = "skipped";
reject(abortError());
return;
}
this._cancelOperation(operation, "signal");
};
signal.addEventListener("abort", onAbort, { once: true });
removeAbortListener = () => signal.removeEventListener("abort", onAbort);
if (signal.aborted) {
onAbort();
}
}
if (watchesDisposal && !started) {
removeDisposalWaiter = this._addDisposalWaiter(() => {
if (!started) {
operation.state = "disposed";
reject(new Error(BRIDGE_DISPOSED_MESSAGE));
}
});
}
});
const raced = Promise.race([ordered, abandonedWhileQueued]);
raced.then(detachWatchers, detachWatchers);
return raced;
}
/**
* Registers a disposal listener for the lifetime of one pending operation and
* returns its remover. A listener set is used rather than a long-lived promise
* because a promise reaction cannot be detached.
*/
_addDisposalWaiter(onDisposed) {
const waiters = this._disposalWaiters;
waiters.add(onDisposed);
return () => {
waiters.delete(onDisposed);
};
}
/**
* True while `operation` is the non-dispose owner of the queue slot. Teardown
* is queued behind it, so it is allowed to finish the work it already started
* even once disposal has been requested.
*/
_isSettleFirstOwner(operation = this._activeOperation) {
return !!(operation && this._activeOperation === operation && operation.kind !== "dispose" && operation.state !== "cancelled" && operation.state !== "disposed");
}
/**
* Lifecycle gate for the helpers that stay off the queue. They must not
* dispatch to a torn-down worker, and must not lazily recreate the runtime
* disposal just released. `disposing` still admits the operation that already
* owns the slot, because disposal settles it first rather than interrupting
* it; once teardown has run, every caller is rejected.
*/
_throwIfDisposed() {
if (this._lifecycleState === "disposed") {
throw new Error(BRIDGE_DISPOSED_MESSAGE);
}
if ((this._disposed || this._lifecycleState === "disposing") && !this._isSettleFirstOwner()) {
throw new Error(BRIDGE_DISPOSED_MESSAGE);
}
}
_notifyDisposalWaiters() {
const waiters = this._disposalWaiters;
if (waiters.size === 0) {
return;
}
this._disposalWaiters = /* @__PURE__ */ new Set();
for (const waiter of waiters) {
waiter();
}
}
/**
* Mirror the direct runtime's model/context state into facade fields so the
* synchronous getters can answer from cache. Reading them straight from the
* runtime would ccall into the singleton core outside the queue, racing
* whichever async operation currently owns it.
*/
_captureDirectRuntimeState() {
if (this._workerProxy || !this._runtime) {
return;
}
try {
this._metadata = this._runtime.getModelMetadata();
this._contextSize = this._runtime.getContextSize();
this._gpuActive = this._runtime.isGpuActive();
this._backendName = this._runtime.getBackendName();
this._supportsVision = this._runtime.supportsVision();
this._supportsAudio = this._runtime.supportsAudio();
} catch (_) {
}
}
/**
* prefetch/evict stay off the queue because they only touch Cache Storage and
* the network. In direct mode their plain-JS cache bookkeeping is copied into
* the facade snapshot. A worker-backed snapshot remains wholly worker-owned so
* a separate cache-only runtime cannot misdescribe the active worker model.
*/
_refreshCacheMetadataSnapshot() {
const runtime = this._runtime;
if (this._workerProxy || !runtime) {
return;
}
this._metadata = {
...this._metadata || {},
"llamadart.webgpu.model_source": String(runtime._modelSource ?? ""),
"llamadart.webgpu.model_cache_state": String(runtime._modelCacheState ?? ""),
"llamadart.webgpu.model_cache_name": String(runtime._modelCacheName ?? ""),
"llamadart.webgpu.runtime_notes": Array.isArray(runtime._runtimeNotes) ? runtime._runtimeNotes.join(";") : ""
};
}
_sanitizeModelLoadOptions(options = {}) {
const source = options && typeof options === "object" ? options : {};
const sanitized = { ...source };
delete sanitized.progressCallback;
delete sanitized.signal;
return sanitized;
}
_createCpuSafeMultimodalLoadOptions(options = {}) {
const sanitized = this._sanitizeModelLoadOptions(options);
sanitized.nGpuLayers = 0;
if (Number.isFinite(Number(sanitized.nCtx)) && Number(sanitized.nCtx) > 4096) {
sanitized.nCtx = 4096;
}
if (!Number.isFinite(Number(sanitized.nThreads)) || Number(sanitized.nThreads) <= 0) {
sanitized.nThreads = 4;
} else {
sanitized.nThreads = Math.min(4, Math.max(1, Math.trunc(Number(sanitized.nThreads))));
}
sanitized.nThreadsBatch = sanitized.nThreads;
if (!Number.isFinite(Number(sanitized.nBatch)) || Number(sanitized.nBatch) <= 0) {
sanitized.nBatch = 128;
} else {
sanitized.nBatch = Math.min(128, Math.max(32, Math.trunc(Number(sanitized.nBatch))));
}
if (!Number.isFinite(Number(sanitized.nUbatch)) || Number(sanitized.nUbatch) <= 0) {
sanitized.nUbatch = Math.min(64, sanitized.nBatch);
} else {
sanitized.nUbatch = Math.min(
sanitized.nBatch,
Math.min(64, Math.max(1, Math.trunc(Number(sanitized.nUbatch))))
);
}
return sanitized;
}
_rememberLoadedModel(url, options = {}) {
const normalizedUrl = cloneModelSource(url);
if (!hasModelSource(normalizedUrl)) {
return;
}
this._loadedModelUrl = normalizedUrl;
this._loadedModelOptions = this._sanitizeModelLoadOptions(options);
this._loadedMmProjUrl = null;
this._forgetLoraAdapters();
this._loadedDraftModel = null;
this._multimodalWorkerCpuMode = this._workerProxy != null;
this._workerModelMissing = false;
if (this._activeOperation) {
this._activeOperation.modelSource = cloneModelSource(normalizedUrl);
this._activeOperation.modelOptions = { ...this._loadedModelOptions };
this._activeOperation.projectorSource = null;
}
}
_forgetLoadedModel() {
this._loadedModelUrl = null;
this._loadedModelOptions = null;
this._loadedMmProjUrl = null;
this._forgetLoraAdapters();
this._loadedDraftModel = null;
this._multimodalWorkerCpuMode = false;
this._workerModelMissing = false;
}
/**
* A failed load can leave its target without any model: the runtime drops
* the previous model once the new download starts. Keep the facade's model
* source only while the target still reports a loaded model, so recovery
* never reloads a model the target no longer holds.
*
* A worker's reply is read from `attempt`, not the facade snapshot: a
* cancelled operation no longer accepts worker state, so the snapshot can
* still describe a model the worker already dropped. Only the bookkeeping is
* corrected here; a cancelled load never publishes the worker's snapshot.
*/
_syncLoadedModelAfterFailedLoad(attempt) {
const proxy = this._workerProxy;
if (!proxy) {
if (!(Number(this._runtime?._modelBytes) > 0)) {
this._forgetLoadedModel();
}
return;
}
const fromCurrentWorker = attempt.proxy === proxy;
const replacedModel = fromCurrentWorker && attempt.loaded;
const state = fromCurrentWorker ? attempt.state : null;
const modelBytes = Number(
(state ? state.metadata : this._metadata)?.["llamadart.webgpu.model_bytes"]
);
if (replacedModel || !(modelBytes > 0)) {
this._forgetLoadedModel();
}
}
_rememberLoadedMmProj(url) {
const normalizedUrl = String(url || "").trim();
if (normalizedUrl.length === 0) {
return;
}
this._loadedMmProjUrl = normalizedUrl;
if (this._activeOperation) {
this._activeOperation.projectorSource = normalizedUrl;
}
}
_hasMediaParts(options = {}) {
return Array.isArray(options?.parts) && options.parts.length > 0;
}
async _replaceWorkerProxyForMultimodalCpuMode() {
this._throwIfDisposed();
const hadWorkerProxy = this._workerProxy != null;
if (hadWorkerProxy) {
const staleProxy = this._workerProxy;
this._advanceWorkerGeneration();
this._workerProxy = null;
this._retireWorkerProxy(staleProxy);
await this._waitForWorkerDisposal();
this._throwIfDisposed();
}
let replacement;
try {
replacement = this._createWorkerProxy();
} catch (error) {
this._disableWorkerFallback(error);
throw error;
}
try {
this._throwIfDisposed();
} catch (error) {
this._retireWorkerProxy(replacement);
throw error;
}
if (!hadWorkerProxy) {
this._advanceWorkerGeneration();
}
this._workerProxy = replacement;
this._recordWorkerGeneration(this._activeOperation, replacement);
this._multimodalWorkerCpuMode = false;
this._workerModelMissing = true;
}
_isRecoverableWorkerFsError(error) {
const text = serializeWorkerError(error).toLowerCase();
if (text.includes("model shard:")) {
return false;
}
return text.includes("fs error") || text.includes("no such file") || text.includes("not found") || text.includes("invalid argument") || text.includes("timed out");
}
_isWorkerRequestTimeoutError(error) {
const text = serializeWorkerError(error).toLowerCase();
return text.includes("worker request timeout") || text.includes("worker init timeout") || text.includes("worker timed out");
}
async _ensureWorkerMultimodalCpuMode() {
this._throwIfDisposed();
if (!this._workerProxy) {
await this._replaceWorkerProxyForMultimodalCpuMode();
}
this._throwIfDisposed();
if (this._multimodalWorkerCpuMode) {
return true;
}
if (!hasModelSource(this._loadedModelUrl)) {
return false;
}
const selectedOptions = this._sanitizeModelLoadOptions(
this._loadedModelOptions || {}
);
const operation = this._activeOperation;
if (operation) {
operation.recoveryAttempted = true;
operation.state = "recovering";
}
const previousDeferredState = this._deferredShadowState;
this._shadowStateTransactionDepth += 1;
this._deferredShadowState = null;
let recoverySucceeded = false;
const applyWorkerSafeMode = () => this._loadRememberedModelIntoWorker(selectedOptions);
try {
await applyWorkerSafeMode();
recoverySucceeded = true;
this._emitBridgeWarn(
"llamadart: multimodal worker prepared in selected backend mode."
);
return true;
} catch (error) {
this._throwIfDisposed();
this._emitBridgeWarn(
`llamadart: multimodal worker setup failed once; restarting worker (${serializeWorkerError(error)}).`
);
await this._replaceWorkerProxyForMultimodalCpuMode();
await applyWorkerSafeMode();
recoverySucceeded = true;
this._emitBridgeWarn(
"llamadart: multimodal worker recovered after restart."
);
return true;
} finally {
this._shadowStateTransactionDepth = Math.max(0, this._shadowStateTransactionDepth - 1);
if (this._shadowStateTransactionDepth === 0) {
if (recoverySucceeded && this._deferredShadowState && this._workerProxy) {
this._applyShadowStateFrom(this._workerProxy, this._deferredShadowState);
}
this._deferredShadowState = previousDeferredState || null;
}
if (operation && this._activeOperation === operation && operation.state === "recovering") {
operation.state = operation.cancelRequested ? "cancelling" : "running";
}
}
}
async _loadRememberedModelIntoWorker(selectedOptions) {
this._throwIfDisposed();
this._dropLoraAdaptersOf(this._workerProxy);
await this._callWorker("loadModelFromUrl", [this._loadedModelUrl, selectedOptions]);
this._throwIfDisposed();
if (typeof this._loadedMmProjUrl === "string" && this._loadedMmProjUrl.length > 0) {
await this._callWorker("loadMultimodalProjector", [this._loadedMmProjUrl]);
}
const draft = this._loadedDraftModel;
if (draft) {
try {
await this._callWorker("loadDraftModel", [draft.url, draft.options]);
} catch (error) {
this._throwIfOperationCancelled(error, "Draft model reload was cancelled.");
this._forgetDraftModelAfterReloadFailure(error);
}
}
this._throwIfDisposed();
await this._restoreLoraAdapters();
this._throwIfDisposed();
this._workerModelMissing = false;
this._loadedModelOptions = selectedOptions;
this._multimodalWorkerCpuMode = true;
}
/**
* Multimodal recovery can replace the worker and then fail to reload the
* model, its projector or its applied LoRA adapters into it. The facade
* still holds that model, but the fresh worker would answer every request
* with "No model loaded" or without the adapters. Reload them first, and
* again on the next call if this fails. A failure here reaches the caller's
* own worker-error handling.
*/
async _restoreWorkerModelIfMissing() {
if (!this._workerProxy || this._workerModelMissing !== true || !hasModelSource(this._loadedModelUrl)) {
return;
}
this._emitBridgeWarn(
"llamadart: bridge worker was restarted without its model; reloading it."
);
await this._loadRememberedModelIntoWorker(
this._sanitizeModelLoadOptions(this._loadedModelOptions || {})
);
}
_isDispatchWorkgroupLimitError(error) {
const text = serializeWorkerError(error).toLowerCase();
return text.includes("dispatch workgroup count") || text.includes("max compute workgroups per dimension") || text.includes("invalid commandbuffer") || text.includes("ggml_webgpu: device error") || text.includes("runtimeerror: aborted") || text.includes("aborted()");
}
// An aborted or trapped Wasm core stays dead. The worker host posts only
// the message text, so a worker-side RuntimeError arrives without its name.
_isWasmCoreAbortError(error) {
if (typeof WebAssembly !== "undefined" && error instanceof WebAssembly.RuntimeError) {
return true;
}
const text = serializeWorkerError(error).toLowerCase();
return text.includes("aborted(") || text.includes("runtimeerror") || text.includes("unreachable") || text.includes("memory access out of bounds") || text.includes("function signature mismatch") || text.includes("program terminated with exit");
}
// A worker request falls back only when the worker can no longer serve
// requests: it crashed, stalled, timed out, never initialized, or its core
// aborted. A core error from a healthy worker (invalid grammar, a context
// limit, an unsupported template, a model without embeddings) is
// deterministic; retrying on the main thread fails the same way and strands
// the session there.
_isWorkerUnusableError(error) {
if (error && typeof error === "object" && error.llamadartWorkerCrash === true) {
return true;
}
return this._isWorkerTimeoutError(error) || this._isWasmCoreAbortError(error) || this._shouldFallbackToMainThread(error);
}
_isWorkerTimeoutError(error) {
if (error && typeof error === "object" && error.llamadartWorkerTimeout === true) {
return true;
}
const text = serializeWorkerError(error).toLowerCase();
return text.includes("worker completion stalled") || text.includes("worker createcompletion stalled") || text.includes("worker timed out") || text.includes("worker timeout");
}
_isCpuModelMode() {
const requestedLayers = Number(this._loadedModelOptions?.nGpuLayers);
if (Number.isFinite(requestedLayers)) {
return requestedLayers === 0;
}
const metadataLayers = Number(this._metadata?.["llamadart.webgpu.n_gpu_layers"]);
if (Number.isFinite(metadataLayers)) {
return metadataLayers === 0;
}
return false;
}
_workerCompletionStallTimeoutMs(options = {}) {
const configured = Number(this._config?.workerGenerationStallTimeoutMs);
if (Number.isFinite(configured) && configured > 0) {
return Math.max(5e3, Math.min(3e5, Math.trunc(configured)));
}
if (!this._hasMediaParts(options)) {
return 9e4;
}
if (this._isCpuModelMode()) {
return 0;
}
return 18e4;
}
async _ensureRuntimeReadyAfterWorkerFallback(options = {}, fallbackError = null) {
this._throwIfDisposed();
await this._waitForWorkerDisposal();
this._throwIfDisposed();
if (!this._runtime) {
this._runtime = this._createRuntime();
}
const operation = this._activeOperation;
operation?.runtimes?.add(this._runtime);
if (operation && operation.state === "running") {
operation.state = "recovering";
}
const forceReloadRequested = options?._llamadartForceRuntimeReload === true;
const mediaPartsRequested = this._hasMediaParts(options);
const textToSpeechRequested = options?._llamadartTextToSpeech === true;
const multimodalRuntimeRequired = mediaPartsRequested || textToSpeechRequested;
const shouldEnsureMultimodalInRuntime = multimodalRuntimeRequired && typeof this._loadedMmProjUrl === "string" && this._loadedMmProjUrl.length > 0;
const workerTimedOut = this._isWorkerTimeoutError(fallbackError);
const dispatchWorkgroupFallback = this._isDispatchWorkgroupLimitError(fallbackError);
const loadedGpuLayers = Number(this._loadedModelOptions?.nGpuLayers);
const metadataGpuLayers = Number(this._metadata?.["llamadart.webgpu.n_gpu_layers"]);
const modelLoadedWithGpu = Number.isFinite(loadedGpuLayers) ? loadedGpuLayers !== 0 : Number.isFinite(metadataGpuLayers) ? metadataGpuLayers !== 0 : true;
const shouldUseCpuMultimodalFallback = multimodalRuntimeRequired && modelLoadedWithGpu && (dispatchWorkgroupFallback || workerTimedOut);
try {
if (Number(this._runtime?._modelBytes) > 0 && !forceReloadRequested && !shouldUseCpuMultimodalFallback) {
await this._ensureRuntimeDraftModel();
if (shouldEnsureMultimodalInRuntime) {
const runtimeSupportsMedia = typeof this._runtime.supportsVision === "function" && this._runtime.supportsVision() || typeof this._runtime.supportsAudio === "function" && this._runtime.supportsAudio();
if (!runtimeSupportsMedia) {
this._throwIfDisposed();
await this._runtime.loadMultimodalProjector(this._loadedMmProjUrl);
this._throwIfDisposed();
if (Array.isArray(this._runtime._runtimeNotes)) {
this._runtime._runtimeNotes.push("worker_fallback_reload_mmproj");
}
}
}
await this._restoreLoraAdapters();
this._throwIfDisposed();
return;
}
if (!hasModelSource(this._loadedModelUrl)) {
return;
}
const loadOptions = shouldUseCpuMultimodalFallback ? this._createCpuSafeMultimodalLoadOptions(this._loadedModelOptions || {}) : this._sanitizeModelLoadOptions(this._loadedModelOptions || {});
const recoverySignal = this._operationSignal(operation) || options.signal || null;
if (recoverySignal) {
loadOptions.signal = recoverySignal;
}
if (shouldUseCpuMultimodalFallback) {
if (textToSpeechRequested) {
this._emitBridgeWarn(
"llamadart: retrying text-to-speech once with CPU fallback after WebGPU failure."
);
} else if (workerTimedOut) {
this._emitBridgeWarn(
"llamadart: retrying multimodal generation with CPU fallback after worker timeout."
);
} else {
this._emitBridgeWarn(
"llamadart: retrying multimodal generation with CPU fallback after WebGPU workgroup limit failure."
);
}
}
if (workerTimedOut) {
this._emitBridgeWarn(
"llamadart: bridge worker completion stalled; restarting generation path on main-thread runtime."
);
}
this._throwIfDisposed();
this._dropLoraAdaptersOf(this._runtime);
await this._runtime.loadModelFromUrl(this._loadedModelUrl, loadOptions);
this._throwIfDisposed();
if (Array.isArray(this._runtime._runtimeNotes)) {
this._runtime._runtimeNotes.push("worker_fallback_reload_model");
if (forceReloadRequested) {
this._runtime._runtimeNotes.push("worker_fallback_reload_forced");
}
if (workerTimedOut) {
this._runtime._runtimeNotes.push("worker_fallback_timeout");
}
if (shouldUseCpuMultimodalFallback) {
this._runtime._runtimeNotes.push("worker_fallback_cpu_multimodal");
if (textToSpeechRequested) {
this._runtime._runtimeNotes.push("worker_fallback_cpu_text_to_speech");
}
}
}
if (shouldEnsureMultimodalInRuntime) {
this._throwIfDisposed();
await this._runtime.loadMultimodalProjector(this._loadedMmProjUrl);
this._throwIfDisposed();
}
await this._restoreLoraAdapters();
this._throwIfDisposed();
await this._ensureRuntimeDraftModel();
} catch (error) {
if (this._disposed || this._lifecycleState === "disposing") {
throw new Error(BRIDGE_DISPOSED_MESSAGE);
}
if (error?.name === "AbortError") {
throw error;
}
try {
await this._runtime?.dispose?.();
} catch (_) {
}
this._runtime = null;
this._metadata = {};
this._contextSize = 0;
this._gpuActive = false;
this._backendName = "";
this._supportsVision = false;
this._supportsAudio = false;
this._loadedModelUrl = null;
this._loadedModelOptions = null;
this._loadedMmProjUrl = null;
this._forgetLoraAdapters();
this._loadedDraftModel = null;
if (!this._disposed && this._lifecycleState === "open" && !this._workerProxy) {
this._runtime = this._createRuntime();
operation?.runtimes?.add(this._runtime);
}
throw error;
} finally {
if (operation && this._activeOperation === operation && operation.state === "recovering") {
operation.state = operation.cancelRequested ? "cancelling" : "running";
}
}
}
_createWorkerProxy() {
return new BridgeWorkerProxy({
moduleUrl: this._workerModuleUrl(),
config: this._workerConfig()
});
}
_shouldUseWorker() {
if (this._config?.disableWorker === true) {
return false;
}
if (typeof Worker === "undefined" || typeof Blob === "undefined" || typeof URL === "undefined" || typeof URL.createObjectURL !== "function") {
return false;
}
if (this._config?.coreModuleFactory != null) {
return false;
}
return true;
}
_workerModuleUrl() {
const candidate = this._config?.workerUrl;
if (typeof candidate === "string" && candidate.trim().length > 0) {
return candidate.trim();
}
try {
return new URL("./llama_webgpu_bridge_worker.js", import.meta.url).toString();
} catch (_) {
return import.meta.url;
}
}
_workerConfig() {
const config = this._config || {};
return {
workerRequestTimeoutMs: config.workerRequestTimeoutMs,
workerInitTimeoutMs: config.workerInitTimeoutMs,
workerModelLoadTimeoutMs: config.workerModelLoadTimeoutMs,
workerMmprojLoadTimeoutMs: config.workerMmprojLoadTimeoutMs,
workerCompletionTimeoutMs: config.workerCompletionTimeoutMs,
workerTextToSpeechTimeoutMs: config.workerTextToSpeechTimeoutMs,
coreInitTimeoutMs: config.coreInitTimeoutMs,
wasmUrl: typeof config.wasmUrl === "string" ? config.wasmUrl : void 0,
wasmUrlMem64: typeof config.wasmUrlMem64 === "string" ? config.wasmUrlMem64 : void 0,
coreModuleUrl: typeof config.coreModuleUrl === "string" ? config.coreModuleUrl : void 0,
coreModuleUrlMem64: typeof config.coreModuleUrlMem64 === "string" ? config.coreModuleUrlMem64 : void 0,
preferMemory64: typeof config.preferMemory64 === "boolean" ? config.preferMemory64 : void 0,
threads: Number(config.threads) > 0 ? Number(config.threads) : void 0,
threadsBatch: Number(config.threadsBatch) > 0 ? Number(config.threadsBatch) : void 0,
threadPoolSize: Number(config.threadPoolSize) > 0 ? Number(config.threadPoolSize) : void 0,
nBatch: Number(config.nBatch) > 0 ? Number(config.nBatch) : void 0,
nUbatch: Number(config.nUbatch) > 0 ? Number(config.nUbatch) : void 0,
nGpuLayers: Number.isFinite(config.nGpuLayers) ? Number(config.nGpuLayers) : void 0,
userAgent: typeof config.userAgent === "string" ? config.userAgent : void 0,
cacheName: typeof config.cacheName === "string" ? config.cacheName : void 0,
remoteFetchThresholdBytes: Number(config.remoteFetchThresholdBytes) > 0 ? Number(config.remoteFetchThresholdBytes) : void 0,
remoteFetchChunkBytes: Number(config.remoteFetchChunkBytes) > 0 ? Number(config.remoteFetchChunkBytes) : void 0,
mediaMaxImagePixels: Number(config.mediaMaxImagePixels) > 0 ? Number(config.mediaMaxImagePixels) : void 0,
mediaMaxImageEdge: Number(config.mediaMaxImageEdge) > 0 ? Number(config.mediaMaxImageEdge) : void 0,
disableImageDownscale: config.disableImageDownscale === true,
allowAutoRemoteFetchBackend: config.allowAutoRemoteFetchBackend === true,
logLevel: Number.isFinite(config.logLevel) ? Number(config.logLevel) : 2
};
}
_applyShadowState(state) {
if (!state || typeof state !== "object") {
return;
}
if (state.metadata && typeof state.metadata === "object") {
this._metadata = state.metadata;
}
if (Number.isFinite(state.contextSize)) {
this._contextSize = Number(state.contextSize);
}
if (typeof state.gpuActive === "boolean") {
this._gpuActive = state.gpuActive;
}
if (typeof state.backendName === "string" && state.backendName.length > 0) {
this._backendName = state.backendName;
}
if (typeof state.supportsVision === "boolean") {
this._supportsVision = state.supportsVision;
}
if (typeof state.supportsAudio === "boolean") {
this._supportsAudio = state.supportsAudio;
}
}
_shouldFallbackToMainThread(error) {
const text = serializeWorkerError(error).toLowerCase();
if (text.includes("aborted(native code called abort())")) {
return false;
}
if (text.includes("array buffer allocation failed")) {
return false;
}
if (text.includes("bad_alloc")) {
return false;
}
if (text.includes("out of memory")) {
return false;
}
if (text.includes("memory access out of bounds")) {
return false;
}
if (text.includes("bridge worker")) {
return true;
}
if (text.includes("worker request failed")) {
return true;
}
if (text.includes("worker request timeout")) {
return true;
}
if (text.includes("worker init timeout")) {
return true;
}
if (text.includes("timed out")) {
return true;
}
if (text.includes("worker proxy is not available")) {
return true;
}
if (text.includes("worker is not initialized")) {
return true;
}
if (text.includes("failed to initialize bridge worker")) {
return true;
}
if (text.includes("script error")) {
return true;
}
return false;
}
_resolvedBridgeLogLevel() {
const configured = Number(this._config?.logLevel);
if (Number.isFinite(configured)) {
return Math.max(0, Math.min(4, Math.trunc(configured)));
}
const runtimeLevel = Number(this._runtime?._logLevel);
if (Number.isFinite(runtimeLevel)) {
return Math.max(0, Math.min(4, Math.trunc(runtimeLevel)));
}
return 2;
}
_shouldEmitBridgeLevel(level) {
const configured = this._resolvedBridgeLogLevel();
const threshold = logThresholdForConfiguredLevel(configured);
if (threshold > 3) {
return false;
}
return logLevelForName(level) >= threshold;
}
_shouldSuppressBridgeWarn(message) {
const text = String(message || "").trim();
if (text.length === 0) {
return false;
}
const configuredWindow = Number(this._config?.warnDedupWindowMs);
const dedupWindowMs = Number.isFinite(configuredWindow) && configuredWindow > 0 ? Math.max(500, Math.min(6e4, Math.trunc(configuredWindow))) : 5e3;
const now = Date.now();
const last = Number(this._bridgeWarnRecent.get(text) || 0);
this._bridgeWarnRecent.set(text, now);
if (this._bridgeWarnRecent.size > 80) {
const staleThreshold = now - dedupWindowMs * 2;
for (const [key, atMs] of this._bridgeWarnRecent.entries()) {
if (Number(atMs) < staleThreshold) {
this._bridgeWarnRecent.delete(key);
}
}
}
return last > 0 && now - last < dedupWindowMs;
}
_emitBridgeWarn(message) {
if (!this._shouldEmitBridgeLevel("warn")) {
return;
}
if (this._shouldSuppressBridgeWarn(message)) {
return;
}
if (this._runtime && typeof this._runtime._emitLogger === "function") {
this._runtime._emitLogger("warn", message);
return;
}
if (typeof console !== "undefined" && typeof console.warn === "function") {
console.warn(message);
}
}
_disableWorkerFallback(error) {
const reason = serializeWorkerError(error);
this._workerFallbackReason = reason;
if (typeof globalThis !== "undefined") {
globalThis.__llamadartBridgeWorkerFallbackReason = reason;
}
this._emitBridgeWarn(
`llamadart: bridge worker unavailable, falling back to main thread (${reason})`
);
if (this._workerProxy) {
const workerProxy = this._workerProxy;
this._advanceWorkerGeneration();
this._workerProxy = null;
this._retireWorkerProxy(workerProxy);
}
this._multimodalWorkerCpuMode = false;
this._workerModelMissing = false;
const activeOperationMayFinish = this._activeOperation && this._activeOperation.kind !== "dispose" && this._activeOperation.state !== "cancelled" && this._activeOperation.state !== "disposed";
if ((this._disposed || this._lifecycleState === "disposing") && !activeOperationMayFinish) {
return;
}
if (!this._runtime) {
this._runtime = this._createRuntime();
this._activeOperation?.runtimes?.add(this._runtime);
}
const note = `worker_fallback:${reason}`;
if (this._runtime && Array.isArray(this._runtime._runtimeNotes) && typeof reason === "string" && reason.length > 0 && !this._runtime._runtimeNotes.includes(note)) {
this._runtime._runtimeNotes.push(note);
}
}
async _callWorker(method, args, onEvent, transferList = []) {
const proxy = this._workerProxy;
if (!proxy) {
throw new Error("Bridge worker proxy is not available");
}
const operation = this._activeOperation;
if (operation) {
this._recordWorkerGeneration(operation, proxy);
}
const guardedEvent = typeof onEvent === "function" ? (event) => {
if (!operation || this._isOperationCallbackCurrent(operation, proxy) || event.event === "usage" && proxy === this._workerProxy) {
onEvent(event);
}
} : void 0;
try {
const response = await proxy.call(
method,
args,
guardedEvent,
transferList,
operation ? { operationId: operation.id, workerGeneration: operation.workerGeneration } : void 0
);
if (response?.state) {
this._acceptWorkerState(operation, proxy, response.state);
}
return response?.value;
} catch (error) {
if (error && typeof error === "object" && error.state) {
this._acceptWorkerState(operation, proxy, error.state);
}
throw error;
}
}
/**
* A worker response may only reach facade state while its proxy is still the
* operation's current owner. Staging it for an open recovery transaction is
* the same commit deferred, so a retired generation must be rejected before
* the transaction branch, not after it.
*/
_acceptWorkerState(operation, proxy, state) {
if (operation && !this._isOperationCallbackCurrent(operation, proxy)) {
return;
}
if (this._shadowStateTransactionDepth > 0) {
this._deferredShadowState = state;
if (operation) {
operation.deferredShadowState = state;
}
return;
}
this._applyShadowStateFrom(proxy, state);
}
/**
* The getters now answer from the facade snapshot, so a response from a proxy
* that has since been replaced must not write into the current worker's state.
*/
_applyShadowStateFrom(proxy, state) {
if (proxy !== this._workerProxy) {
return;
}
this._applyShadowState(state);
}
async loadModelFromUrl(url, options = {}) {
return this._runExclusive(
() => this._loadModelFromUrlUnlocked(url, options),
{
signal: options?.signal,
abortMessage: "Model load was cancelled.",
kind: "load-model"
}
);
}
async _loadModelFromUrlUnlocked(url, options = {}) {
const attempt = { proxy: null, state: null, loaded: false };
try {
return await this._loadModelFromUrlOnTarget(url, options, attempt);
} catch (error) {
this._syncLoadedModelAfterFailedLoad(attempt);
throw error;
}
}
/**
* @param attempt Records the last worker load's proxy and reply for
* `_syncLoadedModelAfterFailedLoad`.
*/
async _loadModelFromUrlOnTarget(url, options, attempt) {
if (!this._workerProxy) {
const runtimeOptions = {
...options,
signal: this._operationSignal() || options.signal || null
};
const result = await this._runtime.loadModelFromUrl(url, runtimeOptions);
this._rememberLoadedModel(url, options);
return result;
}
const invokeWorkerLoad = async () => {
const workerOptions = { ...options };
delete workerOptions.progressCallback;
delete workerOptions.signal;
const proxy = this._workerProxy;
attempt.proxy = proxy;
attempt.state = null;
attempt.loaded = false;
let result;
try {
result = await this._callWorker(
"loadModelFromUrl",
[url, workerOptions],
(event) => {
if (event.event !== "progress") {
return;
}
if (typeof options.progressCallback !== "function") {
return;
}
options.progressCallback(event.payload || {});
}
);
} catch (error) {
attempt.state = error && typeof error === "object" ? error.state || null : null;
throw error;
}
attempt.loaded = true;
this._throwIfCallerCancelled("Model load was cancelled.");
this._rememberLoadedModel(url, workerOptions);
return result;
};
try {
return await invokeWorkerLoad();
} catch (error) {
this._throwIfOperationCancelled(error, "Model load was cancelled.");
if (this._isRecoverableWorkerFsError(error) && !this._isWorkerRequestTimeoutError(error)) {
this._emitBridgeWarn(
`llamadart: worker model-load FS error detected; restarting worker (${serializeWorkerError(error)}).`
);
try {
await this._replaceWorkerProxyForMultimodalCpuMode();
this._forgetLoadedModel();
this._applyShadowState(emptyBridgeState());
return await invokeWorkerLoad();
} catch (retryError) {
this._emitBridgeWarn(
`llamadart: worker model-load retry failed (${serializeWorkerError(retryError)}).`
);
error = retryError;
}
}
if (!this._shouldFallbackToMainThread(error)) {
throw error;
}
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
this._throwIfDisposed();
const result = await this._runtime.loadModelFromUrl(url, {
...options,
signal: this._operationSignal() || options.signal || null
});
this._rememberLoadedModel(url, options);
return result;
}
}
async prefetchModelToCache(url, options = {}) {
this._throwIfDisposed();
if (!this._runtime) {
this._runtime = this._createRuntime();
}
try {
return await this._runtime.prefetchModelToCache(url, options);
} finally {
this._refreshCacheMetadataSnapshot();
}
}
async evictModelFromCache(url, options = {}) {
this._throwIfDisposed();
if (!this._runtime) {
this._runtime = this._createRuntime();
}
try {
return await this._runtime.evictModelFromCache(url, options);
} finally {
this._refreshCacheMetadataSnapshot();
}
}
async createCompletion(prompt, options = {}) {
resolveCompletionSamplingOptions(options);
const onUsage = options?.onUsage;
if (typeof onUsage !== "function") {
return this._runExclusive(
() => this._createCompletionUnlocked(prompt, options),
{
signal: options?.signal,
abortMessage: "Generation was cancelled.",
kind: "generation"
}
);
}
let usage = null;
let text;
try {
text = await this._runExclusive(
() => this._createCompletionUnlocked(prompt, {
...options,
onUsage: (value) => {
usage = value;
}
}),
{
signal: options.signal,
abortMessage: "Generation was cancelled.",
kind: "generation"
}
);
} catch (error) {
if (usage != null && error?.name === "AbortError") {
onUsage(usage);
}
throw error;
}
if (usage != null) {
onUsage(usage);
}
return text;
}
async getCompletionCapabilities() {
return this._runExclusive(
() => this._getCompletionCapabilitiesUnlocked(),
{ kind: "completion-capabilities" }
);
}
async _getCompletionCapabilitiesUnlocked() {
if (!this._workerProxy) {
return this._runtime?.getCompletionCapabilities() ?? { ...NO_COMPLETION_CAPABILITIES };
}
try {
await this._restoreWorkerModelIfMissing();
return await this._callWorker("getCompletionCapabilities", []);
} catch (error) {
this._throwIfOperationCancelled(error, "Completion capability probe was cancelled.");
if (!this._isWorkerUnusableError(error)) {
throw error;
}
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback({}, error);
return this._runtime.getCompletionCapabilities();
}
}
async _createCompletionUnlocked(prompt, options = {}) {
const isWarmup = options?.warmup === true;
const hasRetriedEmptyMultimodal = options?.__llamadartEmptyRetryAttempted === true;
const workerAllowed = this._config?.disableWorker !== true;
if (this._hasMediaParts(options) && workerAllowed) {
const hasWorkerFallback = typeof this._workerFallbackReason === "string" && this._workerFallbackReason.length > 0;
if (hasWorkerFallback && !this._workerProxy && !this._isCpuModelMode()) {
await this._ensureRuntimeReadyAfterWorkerFallback(options, null);
return this._runtime.createCompletion(prompt, options);
}
try {
if (!this._workerProxy) {
await this._replaceWorkerProxyForMultimodalCpuMode();
}
await this._ensureWorkerMultimodalCpuMode();
} catch (error) {
this._throwIfOperationCancelled(error, "Generation was cancelled.");
const reason = serializeWorkerError(error);
if (isWarmup) {
this._emitBridgeWarn(
`llamadart: multimodal warmup skipped after worker setup issue (${reason}).`
);
return "";
}
this._emitBridgeWarn(
`llamadart: unable to prepare multimodal worker CPU mode (${reason}).`
);
if (this._isCpuModelMode()) {
throw new Error(
`CPU multimodal worker setup failed (${reason}). Reload model and retry with a smaller image.`
);
}
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback(options, error);
return this._runtime.createCompletion(prompt, options);
}
}
if (!this._workerProxy) {
return this._runtime.createCompletion(prompt, options);
}
try {
await this._restoreWorkerModelIfMissing();
const workerOptions = { ...options };
delete workerOptions.onToken;
delete workerOptions.onUsage;
delete workerOptions.signal;
delete workerOptions.__llamadartEmptyRetryAttempted;
const speculative = options.speculativeDecoding;
if (speculative != null && typeof speculative === "object" && !Array.isArray(speculative)) {
const resolveCache = (source) => typeof source === "string" && source.length > 0 ? normalizeAbsoluteUrl(source) : source;
workerOptions.speculativeDecoding = {
...speculative,
ngramCacheStatic: resolveCache(speculative.ngramCacheStatic),
ngramCacheDynamic: resolveCache(speculative.ngramCacheDynamic)
};
}
const stallTimeoutMs = this._workerCompletionStallTimeoutMs(options);
let timeoutHandle = null;
let rejectOnStall = null;
const clearStallTimer = () => {
if (timeoutHandle != null) {
globalThis.clearTimeout(timeoutHandle);
timeoutHandle = null;
}
};
const armStallTimer = () => {
clearStallTimer();
if (!Number.isFinite(stallTimeoutMs) || stallTimeoutMs <= 0) {
return;
}
timeoutHandle = globalThis.setTimeout(() => {
const timeoutError = new Error(
`Bridge worker completion stalled for ${stallTimeoutMs}ms.`
);
timeoutError.llamadartWorkerTimeout = true;
this._cancelOperation(this._activeOperation, "worker-timeout");
rejectOnStall?.(timeoutError);
}, stallTimeoutMs);
};
const stallPromise = new Promise((_, reject) => {
rejectOnStall = reject;
});
armStallTimer();
let sawWorkerTokenEvent = false;
try {
const workerResult = await Promise.race([
this._callWorker(
"createCompletion",
[prompt, workerOptions],
(event) => {
if (event.event === "usage") {
if (typeof options.onUsage === "function") {
options.onUsage(event.payload);
}
return;
}
if (event.event !== "token") {
return;
}
armStallTimer();
sawWorkerTokenEvent = true;
if (typeof options.onToken !== "function") {
return;
}
const payload = event.payload || {};
const piece = typeof payload.pieceText === "string" ? payload.pieceText : Uint8Array.from(Array.isArray(payload.piece) ? payload.piece : []);
options.onToken(piece, String(payload.currentText || ""));
}
),
stallPromise
]);
if (this._hasMediaParts(options) && !isWarmup && !sawWorkerTokenEvent && String(workerResult || "").trim().length == 0) {
this._emitBridgeWarn(
"llamadart: multimodal worker produced empty response without token events."
);
if (!hasRetriedEmptyMultimodal) {
this._emitBridgeWarn(
"llamadart: retrying multimodal worker once after empty response."
);
try {
await this._replaceWorkerProxyForMultimodalCpuMode();
await this._ensureWorkerMultimodalCpuMode();
} catch (retrySetupError) {
this._emitBridgeWarn(
`llamadart: multimodal empty-response retry setup failed (${serializeWorkerError(retrySetupError)}).`
);
}
return this._createCompletionUnlocked(prompt, {
...options,
__llamadartEmptyRetryAttempted: true
});
}
}
this._throwIfCallerCancelled("Generation was cancelled.");
return workerResult;
} finally {
clearStallTimer();
}
} catch (error) {
this._throwIfOperationCancelled(error, "Generation was cancelled.");
if (serializeWorkerError(error).includes(INVALID_GRAMMAR_ERROR_TEXT)) {
throw error;
}
if (this._hasMediaParts(options)) {
const reason = serializeWorkerError(error);
if (isWarmup) {
this._emitBridgeWarn(
`llamadart: multimodal warmup skipped after worker request issue (${reason}).`
);
return "";
}
if (this._isCpuModelMode()) {
this._emitBridgeWarn(
`llamadart: CPU multimodal worker request failed (${reason}); skipping main-thread fallback.`
);
throw new Error(
`CPU multimodal request failed (${reason}). Reload model and retry with a smaller image.`
);
}
this._emitBridgeWarn(
`llamadart: multimodal worker request failed (${reason}); falling back to main-thread runtime.`
);
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback(options, error);
return this._runtime.createCompletion(prompt, options);
}
if (!this._isWorkerUnusableError(error)) {
throw error;
}
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback(options, error);
return this._runtime.createCompletion(prompt, options);
}
}
async loadMultimodalProjector(url) {
return this._runExclusive(
() => this._loadMultimodalProjectorUnlocked(url),
{ kind: "projector-load" }
);
}
async _loadMultimodalProjectorUnlocked(url) {
const startedWithWorker = this._workerProxy != null;
const invokeRuntimeLoad = async () => {
if (!this._runtime) {
this._runtime = this._createRuntime();
}
await this._ensureRuntimeReadyAfterWorkerFallback({}, null);
const result = await this._runtime.loadMultimodalProjector(url);
this._rememberLoadedMmProj(url);
this._supportsVision = this._runtime.supportsVision();
this._supportsAudio = this._runtime.supportsAudio();
return result;
};
try {
if (!this._workerProxy) {
return await invokeRuntimeLoad();
}
await this._ensureWorkerMultimodalCpuMode();
const result = await this._callWorker("loadMultimodalProjector", [url]);
this._rememberLoadedMmProj(url);
return result;
} catch (error) {
this._throwIfOperationCancelled(error, "Projector operation was cancelled.");
if (!startedWithWorker) {
throw error;
}
const reason = serializeWorkerError(error);
this._emitBridgeWarn(
`llamadart: multimodal worker setup failed (${reason}).`
);
if (this._isCpuModelMode()) {
try {
await this._replaceWorkerProxyForMultimodalCpuMode();
await this._ensureWorkerMultimodalCpuMode();
const retryResult = await this._callWorker("loadMultimodalProjector", [url]);
this._rememberLoadedMmProj(url);
this._emitBridgeWarn(
"llamadart: CPU multimodal worker setup recovered after worker restart."
);
return retryResult;
} catch (retryError) {
const retryReason = serializeWorkerError(retryError);
throw new Error(
`CPU multimodal projector setup failed (${retryReason}). Reload model and retry with a smaller image.`
);
}
}
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
return invokeRuntimeLoad();
}
}
async unloadMultimodalProjector() {
return this._runExclusive(
() => this._unloadMultimodalProjectorUnlocked(),
{ kind: "projector-unload" }
);
}
async _unloadMultimodalProjectorUnlocked() {
if (!this._workerProxy) {
const result = await this._runtime.unloadMultimodalProjector();
this._loadedMmProjUrl = null;
this._supportsVision = this._runtime.supportsVision();
this._supportsAudio = this._runtime.supportsAudio();
return result;
}
try {
const result = await this._callWorker("unloadMultimodalProjector", []);
this._loadedMmProjUrl = null;
return result;
} catch (error) {
this._throwIfOperationCancelled(error, "Projector operation was cancelled.");
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
const result = await this._runtime.unloadMultimodalProjector();
this._loadedMmProjUrl = null;
this._supportsVision = this._runtime.supportsVision();
this._supportsAudio = this._runtime.supportsAudio();
return result;
}
}
// Loads the remembered draft model into the direct runtime when that
// runtime does not hold it.
async _ensureRuntimeDraftModel() {
const draft = this._loadedDraftModel;
const runtime = this._runtime;
if (!draft || !runtime || runtime._draftModel?.url === draft.url) {
return;
}
this._throwIfDisposed();
try {
await runtime.loadDraftModel(draft.url, {
...draft.options,
signal: this._operationSignal() || void 0
});
} catch (error) {
this._throwIfOperationCancelled(error, "Draft model reload was cancelled.");
this._forgetDraftModelAfterReloadFailure(error);
}
this._throwIfDisposed();
}
// A reloaded target starts without its draft model. A failed draft reload
// leaves none, so draft-* strategies then reject until loadDraftModel runs.
_forgetDraftModelAfterReloadFailure(error) {
this._loadedDraftModel = null;
this._emitBridgeWarn(
`llamadart: draft model reload failed after the model was reloaded (${serializeWorkerError(error)}); call loadDraftModel again.`
);
}
async loadDraftModel(url, options = {}) {
return this._runExclusive(
() => this._loadDraftModelUnlocked(url, options),
{
signal: options?.signal,
abortMessage: "Draft model load was cancelled.",
kind: "draft-model-load"
}
);
}
async _loadDraftModelUnlocked(url, options = {}) {
const absoluteUrl = typeof url === "string" && url.length > 0 ? normalizeAbsoluteUrl(url) : url;
const remembered = {
url: absoluteUrl,
options: typeof options?.useCache === "boolean" ? { useCache: options.useCache } : {}
};
const loadInRuntime = async () => {
this._loadedDraftModel = null;
const info = await this._runtime.loadDraftModel(absoluteUrl, {
...options,
signal: this._operationSignal() || options.signal || void 0
});
this._loadedDraftModel = remembered;
return info;
};
if (!this._workerProxy) {
return loadInRuntime();
}
const workerOptions = { ...options };
delete workerOptions.progressCallback;
delete workerOptions.signal;
try {
await this._restoreWorkerModelIfMissing();
this._loadedDraftModel = null;
const info = await this._callWorker(
"loadDraftModel",
[absoluteUrl, workerOptions],
(event) => {
if (event.event === "progress" && typeof options.progressCallback === "function") {
options.progressCallback(event.payload || {});
}
}
);
this._throwIfCallerCancelled("Draft model load was cancelled.");
this._loadedDraftModel = remembered;
return info;
} catch (error) {
this._throwIfOperationCancelled(error, "Draft model load was cancelled.");
if (!this._isWorkerUnusableError(error)) {
throw error;
}
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback({}, error);
return loadInRuntime();
}
}
async unloadDraftModel() {
return this._runExclusive(
() => this._unloadDraftModelUnlocked(),
{ kind: "draft-model-unload" }
);
}
async _unloadDraftModelUnlocked() {
if (!this._workerProxy) {
await this._runtime?.unloadDraftModel();
this._loadedDraftModel = null;
return;
}
try {
await this._callWorker("unloadDraftModel", []);
this._loadedDraftModel = null;
} catch (error) {
this._throwIfOperationCancelled(error, "Draft model unload was cancelled.");
if (!this._isWorkerUnusableError(error)) {
throw error;
}
this._loadedDraftModel = null;
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._runtime?.unloadDraftModel();
}
}
supportsVision() {
return this._supportsVision;
}
supportsAudio() {
return this._supportsAudio;
}
async getTextToSpeechCapabilities() {
return this._runExclusive(
() => this._getTextToSpeechCapabilitiesUnlocked(),
{ kind: "tts-capabilities" }
);
}
async _getTextToSpeechCapabilitiesUnlocked() {
if (!this._workerProxy) {
return this._runtime.getTextToSpeechCapabilities();
}
try {
await this._restoreWorkerModelIfMissing();
return await this._callWorker("getTextToSpeechCapabilities", []);
} catch (error) {
this._throwIfOperationCancelled(error, "Text-to-speech capability probe was cancelled.");
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback({}, error);
return this._runtime.getTextToSpeechCapabilities();
}
}
async synthesizeSpeech(options = {}) {
return this._runExclusive(
() => this._synthesizeSpeechUnlocked(options),
{
signal: options?.signal,
abortMessage: "Text-to-speech synthesis was cancelled.",
kind: "text-to-speech"
}
);
}
async _synthesizeSpeechUnlocked(options = {}) {
if (options.signal?.aborted) {
throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError");
}
if (!this._workerProxy) {
return this._runtime.synthesizeSpeech(options);
}
try {
await this._restoreWorkerModelIfMissing();
const workerOptions = { ...options };
delete workerOptions.onProgress;
delete workerOptions.signal;
const transferList = [];
const speakerBytes = toUint8Array(options.speakerAudio);
if (speakerBytes && speakerBytes.length > 0) {
const copy = new Uint8Array(speakerBytes);
workerOptions.speakerAudio = copy;
transferList.push(copy.buffer);
}
const result = await this._callWorker(
"synthesizeSpeech",
[workerOptions],
(event) => {
if (event.event === "progress") {
options.onProgress?.(event.payload || {});
}
},
transferList
);
this._throwIfCallerCancelled("Text-to-speech synthesis was cancelled.");
return result;
} catch (error) {
this._throwIfOperationCancelled(error, "Text-to-speech synthesis was cancelled.");
if (error?.name === "AbortError" || options.signal?.aborted || serializeWorkerError(error).toLowerCase().includes("cancel")) {
throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError");
}
const canRetryOnCpu = !this._isCpuModelMode() && (this._isDispatchWorkgroupLimitError(error) || this._isWorkerRequestTimeoutError(error));
if (canRetryOnCpu) {
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback(
{
_llamadartForceRuntimeReload: true,
_llamadartTextToSpeech: true
},
error
);
if (options.signal?.aborted) {
throw new DOMException("Text-to-speech synthesis was cancelled.", "AbortError");
}
return this._runtime.synthesizeSpeech(options);
}
if (this._isWorkerRequestTimeoutError(error)) {
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
}
throw error;
}
}
_decisionHeadMap() {
if (!(this._decisionHeads instanceof Map)) {
this._decisionHeads = /* @__PURE__ */ new Map();
this._nextDecisionHandle = 1;
}
return this._decisionHeads;
}
_registerDecisionHead(owner, info) {
const heads = this._decisionHeadMap();
const handle = this._nextDecisionHandle++;
heads.set(handle, { owner, handle: Number(info?.handle) });
return { ...info, handle };
}
_decisionOwner() {
return this._workerProxy || this._runtime || null;
}
_staleDecisionHeadError(handle) {
return new Error(
`Decision head ${handle} is not loaded; it was freed, its model was unloaded, or the bridge runtime restarted. Load the decision head again.`
);
}
_resolveDecisionHead(handle) {
decisionHandleFrom(handle);
const heads = this._decisionHeadMap();
const entry = heads.get(handle);
if (!entry) {
return null;
}
if (entry.owner !== this._decisionOwner()) {
heads.delete(handle);
return null;
}
return entry;
}
_requireDecisionRuntime() {
if (!this._runtime) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
return this._runtime;
}
/**
* Moves a failed worker's work to the main-thread runtime. Decision heads
* the worker held are gone with it; the model, its projector and the applied
* LoRA adapters are restored so later calls keep working.
*/
async _recoverWorkerOnMainThread(error) {
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback({}, error);
}
async getDecisionCapabilities() {
return this._runExclusive(
() => this._getDecisionCapabilitiesUnlocked(),
{ kind: "decision-capabilities" }
);
}
async _getDecisionCapabilitiesUnlocked() {
if (!this._workerProxy) {
return this._runtime ? this._runtime.getDecisionCapabilities() : { apiVersion: DECISION_API_VERSION, supported: false, reason: "WebGPU core is not initialized" };
}
try {
return await this._callWorker("getDecisionCapabilities", []);
} catch (error) {
this._throwIfOperationCancelled(error, "Decision capability probe was cancelled.");
if (!this._shouldFallbackToMainThread(error)) {
throw error;
}
await this._recoverWorkerOnMainThread(error);
return this._requireDecisionRuntime().getDecisionCapabilities();
}
}
async loadDecisionHead(source, options = {}) {
return this._runExclusive(
() => this._loadDecisionHeadUnlocked(source, options),
{ kind: "decision-head-load" }
);
}
async _loadDecisionHeadUnlocked(source, options = {}) {
const loadInRuntime = async () => {
const runtime = this._requireDecisionRuntime();
return this._registerDecisionHead(
runtime,
await runtime.loadDecisionHead(source, options)
);
};
if (!this._workerProxy) {
return loadInRuntime();
}
const workerOptions = { configJson: options?.configJson ?? null };
let workerSource = source;
const transferList = [];
if (typeof source === "string") {
workerSource = source.length > 0 ? normalizeAbsoluteUrl(source) : source;
} else {
const bytes = bufferSourceBytes(source);
if (bytes) {
const copy = new Uint8Array(bytes);
workerSource = copy;
transferList.push(copy.buffer);
}
}
try {
await this._restoreWorkerModelIfMissing();
const proxy = this._workerProxy;
const info = await this._callWorker(
"loadDecisionHead",
[workerSource, workerOptions],
(event) => {
if (event.event === "progress") {
options?.onProgress?.(event.payload || {});
}
},
transferList
);
return this._registerDecisionHead(proxy, info);
} catch (error) {
this._throwIfOperationCancelled(error, "Decision head load was cancelled.");
if (!this._shouldFallbackToMainThread(error)) {
throw error;
}
await this._recoverWorkerOnMainThread(error);
return loadInRuntime();
}
}
async runDecision(handle, sequences) {
return this._runExclusive(
() => this._runDecisionUnlocked(handle, sequences),
{ kind: "decision-run" }
);
}
async _runDecisionUnlocked(handle, sequences) {
const entry = this._resolveDecisionHead(handle);
if (!entry) {
throw this._staleDecisionHeadError(handle);
}
const normalized = normalizeDecisionSequences(sequences);
if (!this._workerProxy) {
return this._requireDecisionRuntime().runDecision(entry.handle, normalized);
}
try {
return await this._callWorker("runDecision", [entry.handle, normalized]);
} catch (error) {
this._throwIfOperationCancelled(error, "Decision run was cancelled.");
if (!this._shouldFallbackToMainThread(error)) {
throw error;
}
this._decisionHeadMap().delete(handle);
await this._recoverWorkerOnMainThread(error);
throw new Error(
`Decision head ${handle} was lost when the bridge worker failed (${serializeWorkerError(error)}). Load the decision head again.`
);
}
}
async freeDecisionHead(handle) {
return this._runExclusive(
() => this._freeDecisionHeadUnlocked(handle),
{ kind: "decision-head-free" }
);
}
async _freeDecisionHeadUnlocked(handle) {
const entry = this._resolveDecisionHead(handle);
if (!entry) {
return;
}
this._decisionHeadMap().delete(handle);
if (!this._workerProxy) {
await this._requireDecisionRuntime().freeDecisionHead(entry.handle);
return;
}
try {
await this._callWorker("freeDecisionHead", [entry.handle]);
} catch (error) {
this._throwIfOperationCancelled(error, "Decision head free was cancelled.");
if (!this._shouldFallbackToMainThread(error)) {
throw error;
}
await this._recoverWorkerOnMainThread(error);
}
}
// Test doubles skip the constructor, so the LoRA state starts on first use.
_loraAdapterMap() {
if (!(this._loraAdapters instanceof Map)) {
this._loraAdapters = /* @__PURE__ */ new Map();
this._activeLoraScales = /* @__PURE__ */ new Map();
this._nextLoraHandle = 1;
}
return this._loraAdapters;
}
_activeLoraScaleMap() {
this._loraAdapterMap();
return this._activeLoraScales;
}
_forgetLoraAdapters() {
this._loraAdapterMap().clear();
this._activeLoraScaleMap().clear();
}
/**
* Marks the adapters `owner` holds as lost before it loads the model again,
* which frees them. An adapter with a retained source reloads on the next
* restore or set; one without is forgotten.
*/
_dropLoraAdaptersOf(owner) {
for (const [handle, entry] of this._loraAdapterMap()) {
if (owner === null || entry.owner !== owner) {
continue;
}
if (entry.source === null) {
this._loraAdapterMap().delete(handle);
this._activeLoraScaleMap().delete(handle);
} else {
entry.owner = null;
}
}
}
_loraOwner() {
return this._workerProxy || this._runtime || null;
}
_requireLoraRuntime() {
if (!this._runtime) {
throw new Error("No model loaded. Call loadModelFromUrl first.");
}
return this._runtime;
}
_loraAdapterEntry(handle) {
const entry = this._loraAdapterMap().get(handle);
if (!entry) {
throw staleLoraAdapterError(handle);
}
return entry;
}
// Loads an adapter into the current owner: the worker, or the direct runtime.
async _loadLoraAdapterIntoOwner(source, options) {
if (!this._workerProxy) {
const runtime = this._requireLoraRuntime();
const info2 = await runtime.loadLoraAdapter(source, {
...options,
signal: this._operationSignal() || options.signal || null
});
return { owner: runtime, handle: info2.handle };
}
const workerOptions = {};
if (options.useCache !== void 0) {
workerOptions.useCache = options.useCache;
}
if (options.cacheName !== void 0) {
workerOptions.cacheName = options.cacheName;
}
let workerSource = source;
const transferList = [];
if (typeof source !== "string") {
workerSource = new Uint8Array(source);
transferList.push(workerSource.buffer);
}
const proxy = this._workerProxy;
const info = await this._callWorker(
"loadLoraAdapter",
[workerSource, workerOptions],
(event) => {
if (event.event === "progress") {
options.progressCallback?.(event.payload || {});
}
},
transferList
);
return { owner: proxy, handle: Number(info?.handle) };
}
// Reloads an adapter its owner lost, into the current owner.
async _ensureLoraAdapterOnOwner(handle, entry) {
if (entry.owner === this._loraOwner()) {
return;
}
if (entry.source === null) {
this._loraAdapterMap().delete(handle);
this._activeLoraScaleMap().delete(handle);
throw staleLoraAdapterError(handle);
}
const loaded = await this._loadLoraAdapterIntoOwner(entry.source, entry.cacheOptions);
entry.owner = loaded.owner;
entry.handle = loaded.handle;
}
async _callLoraOwner(method, args) {
if (this._workerProxy) {
await this._callWorker(method, args);
return;
}
const runtime = this._requireLoraRuntime();
if (method === "setLoraAdapter") {
await runtime.setLoraAdapter(args[0], args[1]);
} else if (method === "removeLoraAdapter") {
await runtime.removeLoraAdapter(args[0]);
} else {
await runtime.clearLoraAdapters();
}
}
/**
* Makes the current owner apply exactly the facade's active adapters after a
* worker restart or a fallback to the main thread. The owner's own set is
* cleared first, since a runtime that held the model earlier may still apply
* adapters from then. Inactive adapters reload when next set.
*/
async _restoreLoraAdapters() {
const owner = this._loraOwner();
if (![...this._loraAdapterMap().values()].some((entry) => entry.owner !== owner)) {
return;
}
await this._callLoraOwner("clearLoraAdapters", []);
for (const [handle, scale] of [...this._activeLoraScaleMap()]) {
const entry = this._loraAdapterEntry(handle);
await this._ensureLoraAdapterOnOwner(handle, entry);
await this._callLoraOwner("setLoraAdapter", [entry.handle, scale]);
}
}
// Runs a LoRA call on the current owner, moving to the main-thread runtime
// and running it there once if the worker fails.
async _runLoraCall(call, cancelMessage) {
if (!this._workerProxy) {
return call();
}
try {
await this._restoreWorkerModelIfMissing();
return await call();
} catch (error) {
this._throwIfOperationCancelled(error, cancelMessage);
if (!this._shouldFallbackToMainThread(error)) {
throw error;
}
await this._recoverWorkerOnMainThread(error);
return call();
}
}
async getLoraAdapterCapabilities() {
return this._runExclusive(
() => this._getLoraAdapterCapabilitiesUnlocked(),
{ kind: "lora-capabilities" }
);
}
async _getLoraAdapterCapabilitiesUnlocked() {
if (!this._workerProxy && !this._runtime) {
return { apiVersion: LORA_API_VERSION, supported: false, reason: "WebGPU core is not initialized" };
}
return this._runLoraCall(
() => this._workerProxy ? this._callWorker("getLoraAdapterCapabilities", []) : Promise.resolve(this._requireLoraRuntime().getLoraAdapterCapabilities()),
"LoRA capability probe was cancelled."
);
}
async loadLoraAdapter(source, options = {}) {
return this._runExclusive(
() => this._loadLoraAdapterUnlocked(source, options || {}),
{
signal: options?.signal,
abortMessage: LORA_LOAD_ABORT_MESSAGE,
kind: "lora-adapter-load"
}
);
}
async _loadLoraAdapterUnlocked(source, options) {
let normalized;
if (typeof source === "string") {
if (source.length === 0) {
throw new Error("LoRA adapter URL is empty.");
}
normalized = normalizeAbsoluteUrl(source);
} else {
const bytes = bufferSourceBytes(source);
if (!bytes) {
throw new TypeError("LoRA adapter source must be a URL string, an ArrayBuffer or a typed array.");
}
if (bytes.byteLength === 0) {
throw new Error("LoRA adapter bytes are empty.");
}
normalized = bytes;
}
let retained = null;
if (this._config?.disableWorker !== true) {
retained = typeof normalized === "string" ? normalized : new Uint8Array(normalized);
}
const loaded = await this._runLoraCall(
() => this._loadLoraAdapterIntoOwner(retained ?? normalized, options),
LORA_LOAD_ABORT_MESSAGE
);
this._loraAdapterMap();
const handle = this._nextLoraHandle++;
this._loraAdapterMap().set(handle, {
owner: loaded.owner,
handle: loaded.handle,
source: retained,
cacheOptions: { useCache: options.useCache, cacheName: options.cacheName }
});
return { handle };
}
async setLoraAdapter(handle, scale = 1) {
const facadeHandle = loraHandleFrom(handle);
const checkedScale = loraScaleFrom(scale);
return this._runExclusive(
() => this._setLoraAdapterUnlocked(facadeHandle, checkedScale),
{ kind: "lora-adapter-set" }
);
}
async _setLoraAdapterUnlocked(handle, scale) {
const entry = this._loraAdapterEntry(handle);
await this._runLoraCall(async () => {
await this._ensureLoraAdapterOnOwner(handle, entry);
await this._callLoraOwner("setLoraAdapter", [entry.handle, scale]);
}, "LoRA adapter update was cancelled.");
this._activeLoraScaleMap().set(handle, scale);
}
async removeLoraAdapter(handle) {
const facadeHandle = loraHandleFrom(handle);
return this._runExclusive(
() => this._removeLoraAdapterUnlocked(facadeHandle),
{ kind: "lora-adapter-remove" }
);
}
async _removeLoraAdapterUnlocked(handle) {
const entry = this._loraAdapterEntry(handle);
await this._runLoraCall(async () => {
if (entry.owner === this._loraOwner()) {
await this._callLoraOwner("removeLoraAdapter", [entry.handle]);
}
}, "LoRA adapter update was cancelled.");
this._activeLoraScaleMap().delete(handle);
}
async clearLoraAdapters() {
return this._runExclusive(
() => this._clearLoraAdaptersUnlocked(),
{ kind: "lora-adapters-clear" }
);
}
async _clearLoraAdaptersUnlocked() {
await this._runLoraCall(
() => this._callLoraOwner("clearLoraAdapters", []),
"LoRA adapter update was cancelled."
);
this._activeLoraScaleMap().clear();
}
async tokenize(text, addSpecial = true) {
return this._runExclusive(
() => this._tokenizeUnlocked(text, addSpecial),
{ kind: "tokenize" }
);
}
async _tokenizeUnlocked(text, addSpecial = true) {
if (!this._workerProxy) {
return this._runtime.tokenize(text, addSpecial);
}
try {
await this._restoreWorkerModelIfMissing();
return await this._callWorker("tokenize", [text, addSpecial]);
} catch (error) {
this._throwIfOperationCancelled(error, "Tokenization was cancelled.");
if (!this._isWorkerUnusableError(error)) {
throw error;
}
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback({}, error);
return this._runtime.tokenize(text, addSpecial);
}
}
async stateSaveFile(path, tokens = []) {
return this._runExclusive(
() => this._stateSaveFileUnlocked(path, tokens),
{ kind: "state-save-file" }
);
}
async _stateSaveFileUnlocked(path, tokens = []) {
if (!this._workerProxy) {
return this._runtime.stateSaveFile(path, tokens);
}
const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []);
await this._restoreWorkerModelIfMissing();
return this._callWorker("stateSaveFile", [path, normalized]);
}
// The capacity default is resolved inside the slot: evaluating it as a
// parameter default would read runtime state before the queue was taken.
async stateLoadFile(path, tokenCapacity = void 0) {
return this._runExclusive(
() => this._stateLoadFileUnlocked(path, tokenCapacity),
{ kind: "state-load-file" }
);
}
_resolveTokenCapacity(tokenCapacity) {
let numericCapacity;
try {
numericCapacity = Number(tokenCapacity);
} catch {
return this.getContextSize();
}
return numericCapacity > 0 ? Math.trunc(numericCapacity) : this.getContextSize();
}
async _stateLoadFileUnlocked(path, tokenCapacity) {
tokenCapacity = this._resolveTokenCapacity(tokenCapacity);
if (!this._workerProxy) {
return this._runtime.stateLoadFile(path, tokenCapacity);
}
await this._restoreWorkerModelIfMissing();
return this._callWorker("stateLoadFile", [path, tokenCapacity]);
}
async stateSaveBytes(tokens = []) {
return this._runExclusive(
() => this._stateSaveBytesUnlocked(tokens),
{ kind: "state-save-bytes" }
);
}
async _stateSaveBytesUnlocked(tokens = []) {
if (!this._workerProxy) {
return this._runtime.stateSaveBytes(tokens);
}
const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []);
await this._restoreWorkerModelIfMissing();
return this._callWorker("stateSaveBytes", [normalized]);
}
async stateLoadBytes(bytes, tokenCapacity = void 0) {
return this._runExclusive(
() => this._stateLoadBytesUnlocked(bytes, tokenCapacity),
{ kind: "state-load-bytes" }
);
}
async _stateLoadBytesUnlocked(bytes, tokenCapacity) {
tokenCapacity = this._resolveTokenCapacity(tokenCapacity);
if (!this._workerProxy) {
return this._runtime.stateLoadBytes(bytes, tokenCapacity);
}
const normalizedBytes = toUint8Array(bytes);
if (!normalizedBytes || normalizedBytes.length === 0) {
throw new Error("State bytes are empty.");
}
await this._restoreWorkerModelIfMissing();
const transferableBytes = new Uint8Array(normalizedBytes);
return this._callWorker(
"stateLoadBytes",
[transferableBytes, tokenCapacity],
null,
[transferableBytes.buffer]
);
}
async detokenize(tokens, special = false) {
return this._runExclusive(
() => this._detokenizeUnlocked(tokens, special),
{ kind: "detokenize" }
);
}
async _detokenizeUnlocked(tokens, special = false) {
if (!this._workerProxy) {
return this._runtime.detokenize(tokens, special);
}
const normalized = Array.isArray(tokens) ? tokens : Array.from(tokens || []);
try {
await this._restoreWorkerModelIfMissing();
return await this._callWorker("detokenize", [normalized, special]);
} catch (error) {
this._throwIfOperationCancelled(error, "Detokenization was cancelled.");
if (!this._isWorkerUnusableError(error)) {
throw error;
}
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback({}, error);
return this._runtime.detokenize(normalized, special);
}
}
async getEmbeddingCapabilities() {
return this._runExclusive(async () => {
try {
return await this._getEmbeddingCapabilitiesUnlocked();
} catch (error) {
this._throwIfOperationCancelled(error, "Embedding capability probe was cancelled.");
if (!this._workerProxy || !this._isWorkerUnusableError(error)) throw error;
await this._recoverEmbeddingWorker(error);
return this._runtime.getEmbeddingCapabilities();
}
}, { kind: "embedding-capabilities" });
}
async _getEmbeddingCapabilitiesUnlocked() {
if (!this._workerProxy) {
await this._ensureRememberedEmbeddingProjector();
return this._runtime.getEmbeddingCapabilities();
}
try {
await this._restoreWorkerModelIfMissing();
return embeddingCapabilitiesFrom(
await this._callWorker("getEmbeddingCapabilities", []),
embeddingAssetVersion(this._config.workerUrl)
);
} catch (error) {
this._throwIfOperationCancelled(error, "Embedding capability probe was cancelled.");
if (error instanceof Error && /bridge\[method\] is not a function/.test(error.message)) {
throw missingEmbeddingCapability(embeddingAssetVersion(this._config.workerUrl));
}
throw error;
}
}
async embed(text, options = {}) {
const input = typeof text === "string" ? text : snapshotEmbedInput(text);
return this._runExclusive(
() => this._embedUnlocked(input, options),
{ kind: "embedding" }
);
}
_hasEmbeddingMediaParts(input) {
return Array.isArray(input) && input.some((part) => part.type !== "text");
}
async _recoverEmbeddingWorker(error) {
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback({}, error);
await this._ensureRememberedEmbeddingProjector();
}
async _ensureRememberedEmbeddingProjector() {
if (!this._loadedMmProjUrl || !this._runtime?._modelBytes || this._runtime._mmProjPath) return;
this._throwIfDisposed();
try {
await this._runtime.loadMultimodalProjector(this._loadedMmProjUrl);
} catch (error) {
this._throwIfOperationCancelled(error, "Embedding projector restore was cancelled.");
this._emitBridgeWarn("llamadart: embedding projector restore failed; keeping the loaded model.");
}
this._throwIfDisposed();
}
async _embedUnlocked(text, options = {}) {
if (!this._workerProxy) {
if (this._hasEmbeddingMediaParts(text)) await this._ensureRememberedEmbeddingProjector();
return this._runtime.embed(text, options);
}
try {
await this._restoreWorkerModelIfMissing();
if (Array.isArray(text)) await this._getEmbeddingCapabilitiesUnlocked();
return await this._callWorker("embed", [text, options]);
} catch (error) {
this._throwIfOperationCancelled(error, "Embedding was cancelled.");
if (!this._isWorkerUnusableError(error)) {
throw error;
}
await this._recoverEmbeddingWorker(error);
return this._runtime.embed(text, options);
}
}
async embedBatch(texts, options = {}) {
const inputs = Array.from(texts || [], (input) => typeof input === "string" ? input : snapshotEmbedInput(input));
return this._runExclusive(
() => this._embedBatchUnlocked(inputs, options),
{ kind: "embedding-batch" }
);
}
async _embedBatchUnlocked(texts, options = {}) {
const normalized = Array.isArray(texts) ? texts : Array.from(texts || []);
if (!this._workerProxy) {
if (normalized.some((input) => this._hasEmbeddingMediaParts(input))) await this._ensureRememberedEmbeddingProjector();
return this._runtime.embedBatch(normalized, options);
}
try {
await this._restoreWorkerModelIfMissing();
if (normalized.some(Array.isArray)) await this._getEmbeddingCapabilitiesUnlocked();
return await this._callWorker("embedBatch", [normalized, options]);
} catch (error) {
this._throwIfOperationCancelled(error, "Batch embedding was cancelled.");
if (!this._isWorkerUnusableError(error)) {
throw error;
}
await this._recoverEmbeddingWorker(error);
return this._runtime.embedBatch(normalized, options);
}
}
async scoreNextToken(prompt, options = {}) {
return this._runExclusive(
() => this._scoreNextTokenUnlocked(prompt, options),
{ kind: "next-token-scoring" }
);
}
async _scoreNextTokenUnlocked(prompt, options = {}) {
if (!this._workerProxy) {
return this._runtime.scoreNextToken(prompt, options);
}
try {
await this._restoreWorkerModelIfMissing();
return await this._callWorker("scoreNextToken", [prompt, options]);
} catch (error) {
this._throwIfOperationCancelled(error, "Next-token scoring was cancelled.");
if (!this._isWorkerUnusableError(error)) {
throw error;
}
this._disableWorkerFallback(error);
await this._waitForWorkerDisposal();
await this._ensureRuntimeReadyAfterWorkerFallback({}, error);
return this._runtime.scoreNextToken(prompt, options);
}
}
getModelMetadata() {
if (this._workerProxy) {
return {
...this._metadata || {},
"llamadart.webgpu.execution": "worker"
};
}
const workerReason = typeof this._workerFallbackReason === "string" && this._workerFallbackReason.length > 0 ? this._workerFallbackReason : null;
return {
...this._metadata || {},
"llamadart.webgpu.execution": "main-thread",
...workerReason == null ? {} : { "llamadart.webgpu.worker_fallback_reason": workerReason }
};
}
// The synchronous getters answer from the facade snapshot in both modes.
// Reading the direct runtime here would ccall into the singleton core outside
// the queue and race whichever async operation currently owns it; the snapshot
// is refreshed by _captureDirectRuntimeState() while the slot is held.
getContextSize() {
return this._contextSize || 0;
}
isGpuActive() {
return this._gpuActive;
}
getBackendName() {
return this._backendName;
}
setLogLevel(level) {
this._throwIfDisposed();
if (Number.isFinite(level)) {
this._config.logLevel = Math.max(0, Math.min(4, Math.trunc(level)));
}
if (this._workerProxy) {
this._callWorker("setLogLevel", [level]).catch(() => {
});
if (this._runtime) {
this._runtime.setLogLevel(level);
}
return;
}
this._runtime.setLogLevel(level);
}
cancel() {
if (this._lifecycleState === "disposing" || this._lifecycleState === "disposed") {
return;
}
if (this._activeOperation) {
this._cancelOperation(this._activeOperation);
return;
}
if (this._workerProxy) {
this._sendWorkerCancel(this._workerProxy);
}
if (this._runtime) {
try {
this._runtime.cancel();
} catch (_) {
}
}
}
/**
* Marks the bridge disposed synchronously, then tears down on the queue so
* teardown never runs underneath an operation that still owns the runtime.
* Work queued before disposal is rejected with a stable lifecycle error rather
* than dereferencing a torn-down runtime. Not `async`, so repeated calls hand
* back the identical teardown promise.
*/
dispose() {
if (this._disposed) {
return this._disposePromise;
}
this._disposed = true;
this._lifecycleState = "disposing";
let resolveDispose = () => {
};
let rejectDispose = () => {
};
this._disposePromise = new Promise((resolve, reject) => {
resolveDispose = resolve;
rejectDispose = reject;
});
this._notifyDisposalWaiters();
const teardown = this._runExclusive(
() => this._disposeUnlocked(),
{ allowDisposed: true, kind: "dispose" }
);
teardown.then(
() => resolveDispose(),
(error) => rejectDispose(error)
);
return this._disposePromise;
}
async _disposeUnlocked() {
const workerProxy = this._workerProxy;
this._workerProxy = null;
if (workerProxy) {
this._retireWorkerProxy(workerProxy);
}
try {
if (this._retiringWorkerDisposals?.size > 0) {
await this._waitForWorkerDisposal();
}
if (this._runtime) {
const runtime = this._runtime;
this._runtime = null;
await runtime.dispose();
}
} finally {
this._metadata = {};
this._contextSize = 0;
this._gpuActive = false;
this._backendName = "WASM (Prototype bridge)";
this._supportsVision = false;
this._supportsAudio = false;
this._loadedModelUrl = null;
this._loadedModelOptions = null;
this._loadedMmProjUrl = null;
this._loadedDraftModel = null;
this._workerFallbackReason = null;
this._multimodalWorkerCpuMode = false;
this._workerModelMissing = false;
this._decisionHeads?.clear();
this._forgetLoraAdapters();
this._lifecycleState = "disposed";
}
}
async applyChatTemplate(messages, addAssistant = true, customTemplate = null) {
return this._runExclusive(
() => this._applyChatTemplateUnlocked(messages, addAssistant, customTemplate),
{ kind: "template" }
);
}
async _applyChatTemplateUnlocked(messages, addAssistant = true, customTemplate = null) {
if (!this._workerProxy) {
return this._runtime.applyChatTemplate(messages, addAssistant, customTemplate);
}
try {
return await this._callWorker("applyChatTemplate", [messages, addAssistant, customTemplate]);
} catch (error) {
this._throwIfOperationCancelled(error, "Chat template operation was cancelled.");
if (!this._isWorkerUnusableError(error)) {
throw error;
}
this._disableWorkerFallback(error);
return this._runtime.applyChatTemplate(messages, addAssistant, customTemplate);
}
}
};
// js/src/worker_host.ts
var textDecoder = new TextDecoder();
var bridgeWorkerHostInstalled = false;
function installBridgeWorkerHost() {
if (bridgeWorkerHostInstalled) {
return;
}
if (typeof self === "undefined" || typeof self.postMessage !== "function") {
throw new Error("Bridge worker host can only run inside a worker context");
}
bridgeWorkerHostInstalled = true;
let bridge = null;
const postError = (id, error) => {
let state;
try {
state = bridge ? snapshotBridgeState(bridge) : void 0;
} catch (_) {
state = void 0;
}
self.postMessage({
type: "error",
id,
message: serializeWorkerError(error),
...error instanceof EmbeddingError ? { embeddingErrorCode: error.code } : {},
state
});
};
self.onmessage = async (event) => {
const message = event.data || {};
const type = message.type;
const id = message.id ?? 0;
try {
if (type === "init") {
bridge = new LlamaWebGpuBridge({
...message.config || {},
disableWorker: true
});
self.postMessage({ type: "ready" });
return;
}
if (type !== "call") {
return;
}
if (!bridge) {
throw new Error("Bridge worker is not initialized");
}
const method = String(message.method || "");
const args = Array.isArray(message.args) ? message.args : [];
if (method === "loadModelFromUrl") {
const url = args[0];
const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {};
options.progressCallback = (progress) => {
self.postMessage({ type: "event", id, event: "progress", payload: progress || {} });
};
const value2 = await bridge.loadModelFromUrl(url, options);
self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) });
return;
}
if (method === "loadDraftModel") {
const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {};
delete options.signal;
options.progressCallback = (progress) => {
self.postMessage({ type: "event", id, event: "progress", payload: progress || {} });
};
const value2 = await bridge.loadDraftModel(args[0], options);
self.postMessage({ type: "result", id, value: value2 });
return;
}
if (method === "createCompletion") {
const prompt = args[0];
const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {};
delete options.signal;
let usage = null;
options.onUsage = (value3) => {
usage = value3;
};
const tokenEventEncoding = typeof options.tokenEventEncoding === "string" ? String(options.tokenEventEncoding || "").toLowerCase() : "bytes";
const flushMsRaw = Number(options.tokenEventFlushMs);
const tokenEventFlushMs = Number.isFinite(flushMsRaw) && flushMsRaw >= 0 ? Math.max(0, Math.min(200, Math.trunc(flushMsRaw))) : 0;
const flushCharsRaw = Number(options.tokenEventFlushChars);
const tokenEventFlushChars = Number.isFinite(flushCharsRaw) && flushCharsRaw > 0 ? Math.max(1, Math.min(1024, Math.trunc(flushCharsRaw))) : 0;
const shouldEmitCurrentText = options.emitCurrentTextOnToken === true;
let pendingPieceText = "";
let pendingPieceBytes = [];
let pendingPieceByteLength = 0;
let pendingPieceCharLength = 0;
let pendingPieceCharDecoder = tokenEventFlushChars > 0 ? new TextDecoder() : null;
let pendingCurrentText = "";
let flushTimer = null;
const flushTokenTextPayload = () => {
if (pendingPieceText.length === 0) {
return;
}
self.postMessage({
type: "event",
id,
event: "token",
payload: {
pieceText: pendingPieceText,
currentText: shouldEmitCurrentText ? pendingCurrentText : ""
}
});
pendingPieceText = "";
pendingCurrentText = "";
};
const flushTokenBytePayload = () => {
if (pendingPieceByteLength === 0) {
return;
}
const piece = new Uint8Array(pendingPieceByteLength);
let offset = 0;
for (const chunk of pendingPieceBytes) {
piece.set(chunk, offset);
offset += chunk.byteLength;
}
self.postMessage({
type: "event",
id,
event: "token",
payload: {
piece: Array.from(piece),
currentText: shouldEmitCurrentText ? pendingCurrentText : ""
}
});
pendingPieceBytes = [];
pendingPieceByteLength = 0;
pendingPieceCharLength = 0;
pendingCurrentText = "";
};
const flushTokenPayload = () => {
if (tokenEventEncoding === "text") {
flushTokenTextPayload();
return;
}
flushTokenBytePayload();
};
const scheduleTokenFlush = () => {
if (tokenEventFlushMs <= 0 || flushTimer != null) {
return;
}
flushTimer = globalThis.setTimeout(() => {
flushTimer = null;
flushTokenPayload();
}, tokenEventFlushMs);
};
options.onToken = (piece, currentText) => {
if (tokenEventEncoding === "text") {
const pieceText = typeof piece === "string" ? piece : textDecoder.decode(toUint8Array(piece) || new Uint8Array());
if (pieceText.length === 0) {
return;
}
if (tokenEventFlushMs > 0) {
pendingPieceText += pieceText;
if (shouldEmitCurrentText) {
pendingCurrentText = String(currentText || "");
}
if (tokenEventFlushChars > 0 && pendingPieceText.length >= tokenEventFlushChars) {
if (flushTimer != null) {
globalThis.clearTimeout(flushTimer);
flushTimer = null;
}
flushTokenTextPayload();
return;
}
scheduleTokenFlush();
return;
}
self.postMessage({
type: "event",
id,
event: "token",
payload: {
pieceText,
currentText: shouldEmitCurrentText ? String(currentText || "") : ""
}
});
return;
}
const normalizedPieceBytes = toUint8Array(piece) || new Uint8Array();
if (normalizedPieceBytes.byteLength === 0) {
return;
}
if (tokenEventFlushMs > 0) {
const pieceBytes = Array.isArray(piece) ? normalizedPieceBytes : Uint8Array.from(normalizedPieceBytes);
pendingPieceBytes.push(pieceBytes);
pendingPieceByteLength += pieceBytes.byteLength;
if (pendingPieceCharDecoder != null) {
pendingPieceCharLength += pendingPieceCharDecoder.decode(
pieceBytes,
{ stream: true }
).length;
}
if (shouldEmitCurrentText) {
pendingCurrentText = String(currentText || "");
}
if (tokenEventFlushChars > 0 && pendingPieceCharLength >= tokenEventFlushChars) {
if (flushTimer != null) {
globalThis.clearTimeout(flushTimer);
flushTimer = null;
}
flushTokenBytePayload();
return;
}
scheduleTokenFlush();
return;
}
self.postMessage({
type: "event",
id,
event: "token",
payload: {
piece: Array.from(normalizedPieceBytes),
currentText: shouldEmitCurrentText ? String(currentText || "") : ""
}
});
};
let value2;
try {
value2 = await bridge.createCompletion(prompt, options);
} finally {
if (flushTimer != null) {
globalThis.clearTimeout(flushTimer);
flushTimer = null;
}
flushTokenPayload();
}
flushTokenTextPayload();
if (usage != null) {
self.postMessage({ type: "event", id, event: "usage", payload: usage });
}
self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) });
return;
}
if (method === "loadMultimodalProjector") {
const value2 = await bridge.loadMultimodalProjector(args[0]);
self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) });
return;
}
if (method === "synthesizeSpeech") {
const options = args[0] && typeof args[0] === "object" ? { ...args[0] } : {};
delete options.signal;
options.onProgress = (progress) => {
self.postMessage({ type: "event", id, event: "progress", payload: progress || {} });
};
const value2 = await bridge.synthesizeSpeech(options);
const transfers = value2?.pcm?.buffer instanceof ArrayBuffer ? [value2.pcm.buffer] : [];
self.postMessage(
{ type: "result", id, value: value2, state: snapshotBridgeState(bridge) },
transfers
);
return;
}
if (method === "loadDecisionHead") {
const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {};
options.onProgress = (progress) => {
self.postMessage({ type: "event", id, event: "progress", payload: progress || {} });
};
const value2 = await bridge.loadDecisionHead(args[0], options);
self.postMessage({ type: "result", id, value: value2 });
return;
}
if (method === "loadLoraAdapter") {
const options = args[1] && typeof args[1] === "object" ? { ...args[1] } : {};
delete options.signal;
options.progressCallback = (progress) => {
self.postMessage({ type: "event", id, event: "progress", payload: progress || {} });
};
const value2 = await bridge.loadLoraAdapter(args[0], options);
self.postMessage({ type: "result", id, value: value2 });
return;
}
if (method === "runDecision") {
const value2 = await bridge.runDecision(args[0], args[1]);
const transfers = [];
for (const output of Array.isArray(value2) ? value2 : []) {
for (const values of [output?.logits, output?.actLogits]) {
const buffer = values?.buffer;
if (buffer instanceof ArrayBuffer && !transfers.includes(buffer)) {
transfers.push(buffer);
}
}
}
self.postMessage({ type: "result", id, value: value2 }, transfers);
return;
}
if (method === "unloadMultimodalProjector") {
const value2 = await bridge.unloadMultimodalProjector();
self.postMessage({ type: "result", id, value: value2, state: snapshotBridgeState(bridge) });
return;
}
if (method === "stateSaveBytes") {
const value2 = await bridge.stateSaveBytes(args[0]);
const transfers = value2 && value2.buffer instanceof ArrayBuffer ? [value2.buffer] : [];
self.postMessage({ type: "result", id, value: value2 }, transfers);
return;
}
if (method === "stateLoadBytes") {
const value2 = await bridge.stateLoadBytes(args[0], args[1]);
self.postMessage({ type: "result", id, value: value2 });
return;
}
if (method === "dispose") {
const value2 = await bridge.dispose();
self.postMessage({
type: "result",
id,
value: value2,
state: emptyBridgeState()
});
return;
}
const value = await bridge[method](
...args || []
);
self.postMessage({ type: "result", id, value });
} catch (error) {
postError(id, error);
}
};
}
function shouldAutoBootBridgeWorkerHost() {
if (typeof WorkerGlobalScope === "undefined" || !(globalThis instanceof WorkerGlobalScope)) {
return false;
}
try {
const href = String(globalThis.location?.href || "");
if (!href) {
return false;
}
const url = new URL(href);
return url.searchParams.get(bridgeWorkerModeParam) === "1";
} catch (_) {
return false;
}
}
function enableBridgeWorkerHost() {
installBridgeWorkerHost();
}
// js/src/llama_webgpu_bridge.js
if (shouldAutoBootBridgeWorkerHost()) {
installBridgeWorkerHost();
}
if (typeof window !== "undefined") {
const browserWindow = (
/** @type {Window & typeof globalThis & { LlamaWebGpuBridge?: typeof LlamaWebGpuBridge }} */
window
);
if (!browserWindow.LlamaWebGpuBridge) {
browserWindow.LlamaWebGpuBridge = LlamaWebGpuBridge;
}
}
export {
EmbeddingError,
LlamaWebGpuBridge,
enableBridgeWorkerHost
};