// IndexedDB model library. // // v2 layout (this file): // models store, keyPath "id" -> one record per model, id = uuid // modelChunks store, keyPath [modelId, index] -> { modelId, index, data: ArrayBuffer } // // Why a uuid primary key: one browser should hold many models, the same file may // legitimately be registered twice under different names, and records must stay // addressable after a rename. Nothing user-visible keys off the id. // // Why chunks instead of one blob: // 1. pause/resume - a Range request continues from meta.received without // rewriting hundreds of MB already stored; // 2. loading - chunks stream straight into the Emscripten FS, so no second // full-size buffer is assembled in JS memory; // 3. hashing - the same generator feeds the SHA-256, 4 MiB at a time. // // A record with source "file" normally has *no* chunks: the bytes stay on the // user's disk and are read in slices at load time. `copied` marks the opt-in // case where the user asked for a copy to live in the browser instead. export const DB_NAME = "llmjs-models"; export const DB_VERSION = 2; export const CHUNK_SIZE = 4 * 1024 * 1024; const MODEL_STORE = "models"; const CHUNK_STORE = "modelChunks"; let dbPromise = null; export function newModelId() { if (globalThis.crypto && typeof crypto.randomUUID === "function") return crypto.randomUUID(); const bytes = crypto.getRandomValues(new Uint8Array(16)); return [...bytes].map((b) => b.toString(16).padStart(2, "0")).join(""); } export function nameFromUrl(url) { try { const base = new URL(url, "http://localhost/").pathname.split("/").filter(Boolean).pop() || "model"; return base.replace(/\.gguf$/i, ""); } catch (_) { return "model"; } } // --- v1 -> v2 --------------------------------------------------------------- // v1 keyed records by url (out-of-line keys) and chunks by "url|00000000". // Migration runs inside the versionchange transaction: copy the old stores into // the new ones, then drop the old ones. function migrateV1(db, tx, models, chunks) { const v1meta = tx.objectStore("meta"); const v1chunks = tx.objectStore("chunks"); const ids = new Map(); const metaCursor = v1meta.openCursor(); metaCursor.onsuccess = () => { const cursor = metaCursor.result; if (cursor) { const old = cursor.value || {}; const id = newModelId(); if (old.url) ids.set(old.url, id); models.put({ id, name: nameFromUrl(old.url || ""), source: "url", url: old.url || null, fileName: null, total: old.total || 0, received: old.received || 0, complete: !!old.complete, copied: true, // v1 only ever stored downloaded bytes sha256: null, expectedSha256: null, created: old.updated || Date.now(), updated: old.updated || Date.now(), }); cursor.continue(); return; } const chunkCursor = v1chunks.openCursor(); chunkCursor.onsuccess = () => { const c = chunkCursor.result; if (c) { const key = String(c.key); const bar = key.lastIndexOf("|"); const modelId = ids.get(key.slice(0, bar)); const index = Number(key.slice(bar + 1)); if (modelId !== undefined && Number.isFinite(index)) { chunks.put({ modelId, index, data: c.value }); } c.continue(); return; } db.deleteObjectStore("meta"); db.deleteObjectStore("chunks"); }; }; } export function openDB() { if (!dbPromise) { dbPromise = new Promise((resolve, reject) => { if (typeof indexedDB === "undefined") { reject(new Error("IndexedDB is not available in this context")); return; } const req = indexedDB.open(DB_NAME, DB_VERSION); req.onupgradeneeded = (event) => { const db = req.result; const tx = req.transaction; const models = db.objectStoreNames.contains(MODEL_STORE) ? tx.objectStore(MODEL_STORE) : db.createObjectStore(MODEL_STORE, { keyPath: "id" }); const chunks = db.objectStoreNames.contains(CHUNK_STORE) ? tx.objectStore(CHUNK_STORE) : db.createObjectStore(CHUNK_STORE, { keyPath: ["modelId", "index"] }); if (event.oldVersion === 1) migrateV1(db, tx, models, chunks); }; req.onsuccess = () => resolve(req.result); req.onerror = () => reject(req.error); }); } return dbPromise; } const result = (req) => new Promise((res, rej) => { req.onsuccess = () => res(req.result); req.onerror = () => rej(req.error); }); const completed = (tx) => new Promise((res, rej) => { tx.oncomplete = () => res(); tx.onerror = () => rej(tx.error); tx.onabort = () => rej(tx.error); }); const chunkRange = (modelId, from = 0) => IDBKeyRange.bound([modelId, from], [modelId, Number.MAX_SAFE_INTEGER]); // --- models ---------------------------------------------------------------- export async function getModel(id) { const db = await openDB(); return result(db.transaction(MODEL_STORE, "readonly").objectStore(MODEL_STORE).get(id)); } export async function putModel(model) { const db = await openDB(); const record = { ...model, updated: Date.now() }; const tx = db.transaction(MODEL_STORE, "readwrite"); tx.objectStore(MODEL_STORE).put(record); await completed(tx); return record; } export async function updateModel(id, patch) { const model = await getModel(id); if (!model) throw new Error(`unknown model ${id}`); return putModel({ ...model, ...patch, id }); } // most recently touched first export async function listModels() { const db = await openDB(); const all = await result(db.transaction(MODEL_STORE, "readonly").objectStore(MODEL_STORE).getAll()); return all.sort((a, b) => (b.updated || 0) - (a.updated || 0)); } export async function findModelByUrl(url) { const all = await listModels(); return all.find((m) => m.url === url); } // --- chunks ---------------------------------------------------------------- export async function putChunk(modelId, index, buffer) { const db = await openDB(); const tx = db.transaction(CHUNK_STORE, "readwrite"); tx.objectStore(CHUNK_STORE).put({ modelId, index, data: buffer }); await completed(tx); } export async function getChunk(modelId, index) { const db = await openDB(); const row = await result(db.transaction(CHUNK_STORE, "readonly").objectStore(CHUNK_STORE).get([modelId, index])); return row ? row.data : undefined; } export async function chunkCount(modelId) { const db = await openDB(); return result(db.transaction(CHUNK_STORE, "readonly").objectStore(CHUNK_STORE).count(chunkRange(modelId))); } // ascending chunk indexes currently stored for this model export async function chunkIndexes(modelId) { const db = await openDB(); const keys = await result(db.transaction(CHUNK_STORE, "readonly").objectStore(CHUNK_STORE).getAllKeys(chunkRange(modelId))); return keys.map((k) => (Array.isArray(k) ? k[1] : Number(k))).sort((a, b) => a - b); } // Yields the stored bytes in order. Reads in small batches so the transaction // is never held open across awaits (which would let it auto-commit) and so a // 4 GB model never lands in memory as one buffer. export async function* iterateChunks(modelId, batch = 4) { let start = 0; for (;;) { const db = await openDB(); const rows = await result( db.transaction(CHUNK_STORE, "readonly").objectStore(CHUNK_STORE) .getAll(chunkRange(modelId, start), batch), ); if (!rows.length) return; const from = start; for (let i = 0; i < rows.length; i++) { yield { index: from + i, data: rows[i].data }; } if (rows.length < batch) return; start += rows.length; } } export async function clearChunks(modelId) { const db = await openDB(); const tx = db.transaction(CHUNK_STORE, "readwrite"); const store = tx.objectStore(CHUNK_STORE); let deleted = 0; const req = store.openCursor(chunkRange(modelId)); req.onsuccess = () => { const cursor = req.result; if (!cursor) return; cursor.delete(); deleted++; cursor.continue(); }; await completed(tx); return deleted; } // --- lifecycle ------------------------------------------------------------- // removes the record and everything stored for it (the "delete" button) export async function deleteModel(modelId) { const chunks = await clearChunks(modelId); const db = await openDB(); const tx = db.transaction(MODEL_STORE, "readwrite"); tx.objectStore(MODEL_STORE).delete(modelId); await completed(tx); return chunks; } export async function clearAll() { const models = await listModels(); for (const m of models) await deleteModel(m.id); return models.length; } export async function storageEstimate() { if (navigator.storage && navigator.storage.estimate) { try { return await navigator.storage.estimate(); } catch (_) { /* ignore */ } } return null; }