// One-time online setup + offline status. Everything ends up in this origin's // Cache Storage: the app shell and CheerpX (via sw.js), the disk image chunks // (stored decompressed by downloadDisk), and the model (by transformers.js). import { CX_BASE, CX_FILES, CACHE_CX, CACHE_DISK, CACHE_MODEL, DISK_BASE, modelFiles, hubUrl, effectiveModel, } from "./config.js"; import { fetchToCache, clearPartial, PARTIAL_CACHE } from "./fast-download.js"; const scopeUrl = (p) => new URL(p, document.baseURI).href; const MANIFEST_KEY = () => scopeUrl("__vm__/manifest.json"); const chunkKey = (i) => scopeUrl("__vm__/chunk/" + i); /** Registers sw.js and makes sure this page is controlled by it (reloads once on first visit). */ export async function ensureServiceWorker() { if (!("serviceWorker" in navigator)) { throw new Error("Service workers are unavailable here (private window or unsupported browser), so offline mode can't work."); } const reg = await navigator.serviceWorker.register("./sw.js", { scope: "./" }); if (!navigator.serviceWorker.controller) { await navigator.serviceWorker.ready; if (!sessionStorage.getItem("lwa-sw-reload")) { sessionStorage.setItem("lwa-sw-reload", "1"); location.reload(); await new Promise(() => {}); // wait for the reload } } sessionStorage.removeItem("lwa-sw-reload"); return reg; } export async function requestPersistence() { try { return await navigator.storage?.persist?.(); } catch { return false; } } export async function storageEstimate() { try { return await navigator.storage.estimate(); } catch { return null; } } // ---------- status ---------- export async function cheerpxStatus() { const cache = await caches.open(CACHE_CX); const have = await Promise.all(CX_FILES.map((f) => cache.match(CX_BASE + f))); return { ready: have.every(Boolean), have: have.filter(Boolean).length, total: CX_FILES.length }; } export async function installedDisk() { const cache = await caches.open(CACHE_DISK); const res = await cache.match(MANIFEST_KEY()); return res ? res.json() : null; } export async function modelStatus(model) { model = effectiveModel(model); // the q4 build without f16 shaders const cache = await caches.open(CACHE_MODEL); const have = await Promise.all(modelFiles(model).map((f) => cache.match(hubUrl(model, f)))); return { ready: have.every(Boolean) }; } /** Bytes this model's files take in Cache Storage (0 if not downloaded). */ export async function modelCachedBytes(model) { const cache = await caches.open(CACHE_MODEL); const prefix = `https://huggingface.co/${model.repo}/`; let total = 0; for (const req of await cache.keys()) { if (!req.url.startsWith(prefix)) continue; const res = await cache.match(req); total += Number(res?.headers.get("Content-Length")) || 0; } return total; } /** * Downloads every file a local model needs into Cache Storage (the cache transformers.js * reads). Weight files use parallel, resumable range requests (fast-download.js); the * small config/tokenizer files are fetched all at once. Files already cached are skipped. * onProgress(loaded, total) in bytes; signal aborts (finished pieces are kept for a resume). */ export async function downloadModel(model, { onProgress, signal } = {}) { model = effectiveModel(model); const cache = await caches.open(CACHE_MODEL); const files = modelFiles(model); const sizes = new Map(); let loaded = 0; const report = () => { let known = 0; for (const v of sizes.values()) known += v; onProgress?.(loaded, Math.max(model.bytes, known, loaded)); }; const todo = []; for (const f of files) { const hit = await cache.match(hubUrl(model, f)); if (hit) { const n = Number(hit.headers.get("Content-Length")) || 0; sizes.set(f, n); loaded += n; } else todo.push(f); } report(); const get = (f) => fetchToCache(cache, hubUrl(model, f), { signal, onSize: (n) => { sizes.set(f, n); report(); }, onBytes: (d) => { loaded += d; report(); }, }); const big = todo.filter((f) => f.startsWith("onnx/")); const small = todo.filter((f) => !f.startsWith("onnx/")); // Small files in parallel with the weights; weight files one after another, each // using the full pool of connections. await Promise.all([ Promise.all(small.map(get)), (async () => { for (const f of big) await get(f); })(), ]); report(); } /** * Deletes downloaded files of models that are no longer in the catalog (e.g. after a * model was dropped from the list), including pieces of unfinished downloads. * Returns the number of bytes freed. */ export async function removeOrphanedModels(knownRepos) { const known = new Set(knownRepos); const repoOf = (url) => /^https:\/\/huggingface\.co\/([^/]+\/[^/]+)\/resolve\//.exec(url)?.[1]; let freed = 0; for (const name of [CACHE_MODEL, PARTIAL_CACHE]) { const cache = await caches.open(name); for (const req of await cache.keys()) { const repo = repoOf(req.url); if (!repo || known.has(repo)) continue; const res = await cache.match(req); freed += Number(res?.headers.get("Content-Length")) || 0; await cache.delete(req); } } return freed; } /** Deletes one model's downloaded files. */ export async function removeModel(model) { const cache = await caches.open(CACHE_MODEL); const prefix = `https://huggingface.co/${model.repo}/`; for (const req of await cache.keys()) if (req.url.startsWith(prefix)) await cache.delete(req); await clearPartial(prefix); // pieces of an unfinished download } export async function fetchRemoteManifest() { const res = await fetch(DISK_BASE + "manifest.json", { cache: "no-store" }); if (!res.ok) throw new Error(`disk manifest: HTTP ${res.status}`); return res.json(); } // ---------- downloads ---------- export async function downloadCheerpX(onProgress) { let done = 0; for (const f of CX_FILES) { // Goes through sw.js, which stores it in the CheerpX cache. const res = await fetch(CX_BASE + f, { mode: "cors" }); if (!res.ok && res.status !== 204) throw new Error(`CheerpX ${f}: HTTP ${res.status}`); await res.arrayBuffer(); onProgress?.(++done / CX_FILES.length); } } /** * Downloads the chunked disk image. Resumable: chunks already stored are skipped. * The manifest is written last, so its presence means the image is complete. */ export async function downloadDisk(onProgress, { concurrency = 4 } = {}) { const manifest = await fetchRemoteManifest(); const cache = await caches.open(CACHE_DISK); const old = await installedDisk(); if (old && old.id !== manifest.id) { await caches.delete(CACHE_DISK); // different image: start clean } const disk = await caches.open(CACHE_DISK); const todo = []; let doneBytes = 0; for (let i = 0; i < manifest.chunks.length; i++) { if (!manifest.chunks[i]) continue; if (await disk.match(chunkKey(i))) doneBytes += manifest.chunks[i]; else todo.push(i); } const total = manifest.downloadBytes; onProgress?.(doneBytes, total); let next = 0; async function worker() { while (next < todo.length) { const i = todo[next++]; const name = `chunk-${String(i).padStart(5, "0")}.gz`; let attempt = 0; for (;;) { try { const res = await fetch(DISK_BASE + name, { cache: "no-store" }); if (!res.ok) throw new Error(`${name}: HTTP ${res.status}`); // Stored as downloaded (gzipped): the service worker decompresses a chunk when the // VM first reads it. Keeps Linux at ~0.6 GB of browser storage instead of ~2 GB. const body = new Uint8Array(await res.arrayBuffer()); await disk.put(chunkKey(i), new Response(body, { headers: { "Content-Type": "application/octet-stream" } })); doneBytes += manifest.chunks[i]; onProgress?.(doneBytes, total); break; } catch (err) { if (++attempt >= 4) throw err; await new Promise((r) => setTimeout(r, 1000 * attempt)); } } } } await Promise.all(Array.from({ length: concurrency }, worker)); await cache.put(MANIFEST_KEY(), new Response(JSON.stringify(manifest), { headers: { "Content-Type": "application/json" } })); navigator.serviceWorker.controller?.postMessage({ type: "disk-updated" }); return manifest; } function deleteDb(name) { return new Promise((resolve) => { const req = indexedDB.deleteDatabase(name); req.onsuccess = req.onerror = req.onblocked = () => resolve(); }); } /** Wipes the VM's saved changes. CheerpX names the IndexedDB database "cjFS_//". */ export async function resetVmStorage(overlayName) { const dbs = (await indexedDB.databases?.()) ?? []; const names = dbs.map((d) => d.name).filter((n) => n?.includes(overlayName)); for (const n of names.length ? names : ["cjFS_/" + overlayName + "/"]) await deleteDb(n); } /** Removes everything this app stored (model, disk, CheerpX, VM files). */ export async function uninstallAll() { for (const k of await caches.keys()) { if (k.startsWith("lwa-") || k === CACHE_MODEL || k.startsWith("transformers")) await caches.delete(k); } const dbs = (await indexedDB.databases?.()) ?? []; for (const db of dbs) if (db.name?.includes("lwa-")) await deleteDb(db.name); }