Spaces:
Running
Running
Download js/setup.js from ereniko/LocalWebAgent: direct link, hf CLI and curl.
- Browser
- Download file 9.33 kB
-
https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/js/setup.js
- Command line
-
hf download hf://spaces/ereniko/LocalWebAgent/js/setup.js
-
curl -L -o setup.js https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/js/setup.js
9.33 kB
| // One-time online setup + offline status. Everything ends up in this origin's | |
| // Cache Storage: the app shell and CheerpX (via sw.js), the disk image chunks | |
| // (stored decompressed by downloadDisk), and the model (by transformers.js). | |
| import { | |
| CX_BASE, CX_FILES, CACHE_CX, CACHE_DISK, CACHE_MODEL, DISK_BASE, modelFiles, hubUrl, effectiveModel, | |
| } from "./config.js"; | |
| import { fetchToCache, clearPartial, PARTIAL_CACHE } from "./fast-download.js"; | |
| const scopeUrl = (p) => new URL(p, document.baseURI).href; | |
| const MANIFEST_KEY = () => scopeUrl("__vm__/manifest.json"); | |
| const chunkKey = (i) => scopeUrl("__vm__/chunk/" + i); | |
| /** Registers sw.js and makes sure this page is controlled by it (reloads once on first visit). */ | |
| export async function ensureServiceWorker() { | |
| if (!("serviceWorker" in navigator)) { | |
| throw new Error("Service workers are unavailable here (private window or unsupported browser), so offline mode can't work."); | |
| } | |
| const reg = await navigator.serviceWorker.register("./sw.js", { scope: "./" }); | |
| if (!navigator.serviceWorker.controller) { | |
| await navigator.serviceWorker.ready; | |
| if (!sessionStorage.getItem("lwa-sw-reload")) { | |
| sessionStorage.setItem("lwa-sw-reload", "1"); | |
| location.reload(); | |
| await new Promise(() => {}); // wait for the reload | |
| } | |
| } | |
| sessionStorage.removeItem("lwa-sw-reload"); | |
| return reg; | |
| } | |
| export async function requestPersistence() { | |
| try { return await navigator.storage?.persist?.(); } catch { return false; } | |
| } | |
| export async function storageEstimate() { | |
| try { return await navigator.storage.estimate(); } catch { return null; } | |
| } | |
| // ---------- status ---------- | |
| export async function cheerpxStatus() { | |
| const cache = await caches.open(CACHE_CX); | |
| const have = await Promise.all(CX_FILES.map((f) => cache.match(CX_BASE + f))); | |
| return { ready: have.every(Boolean), have: have.filter(Boolean).length, total: CX_FILES.length }; | |
| } | |
| export async function installedDisk() { | |
| const cache = await caches.open(CACHE_DISK); | |
| const res = await cache.match(MANIFEST_KEY()); | |
| return res ? res.json() : null; | |
| } | |
| export async function modelStatus(model) { | |
| model = effectiveModel(model); // the q4 build without f16 shaders | |
| const cache = await caches.open(CACHE_MODEL); | |
| const have = await Promise.all(modelFiles(model).map((f) => cache.match(hubUrl(model, f)))); | |
| return { ready: have.every(Boolean) }; | |
| } | |
| /** Bytes this model's files take in Cache Storage (0 if not downloaded). */ | |
| export async function modelCachedBytes(model) { | |
| const cache = await caches.open(CACHE_MODEL); | |
| const prefix = `https://huggingface.co/${model.repo}/`; | |
| let total = 0; | |
| for (const req of await cache.keys()) { | |
| if (!req.url.startsWith(prefix)) continue; | |
| const res = await cache.match(req); | |
| total += Number(res?.headers.get("Content-Length")) || 0; | |
| } | |
| return total; | |
| } | |
| /** | |
| * Downloads every file a local model needs into Cache Storage (the cache transformers.js | |
| * reads). Weight files use parallel, resumable range requests (fast-download.js); the | |
| * small config/tokenizer files are fetched all at once. Files already cached are skipped. | |
| * onProgress(loaded, total) in bytes; signal aborts (finished pieces are kept for a resume). | |
| */ | |
| export async function downloadModel(model, { onProgress, signal } = {}) { | |
| model = effectiveModel(model); | |
| const cache = await caches.open(CACHE_MODEL); | |
| const files = modelFiles(model); | |
| const sizes = new Map(); | |
| let loaded = 0; | |
| const report = () => { | |
| let known = 0; | |
| for (const v of sizes.values()) known += v; | |
| onProgress?.(loaded, Math.max(model.bytes, known, loaded)); | |
| }; | |
| const todo = []; | |
| for (const f of files) { | |
| const hit = await cache.match(hubUrl(model, f)); | |
| if (hit) { const n = Number(hit.headers.get("Content-Length")) || 0; sizes.set(f, n); loaded += n; } | |
| else todo.push(f); | |
| } | |
| report(); | |
| const get = (f) => fetchToCache(cache, hubUrl(model, f), { | |
| signal, | |
| onSize: (n) => { sizes.set(f, n); report(); }, | |
| onBytes: (d) => { loaded += d; report(); }, | |
| }); | |
| const big = todo.filter((f) => f.startsWith("onnx/")); | |
| const small = todo.filter((f) => !f.startsWith("onnx/")); | |
| // Small files in parallel with the weights; weight files one after another, each | |
| // using the full pool of connections. | |
| await Promise.all([ | |
| Promise.all(small.map(get)), | |
| (async () => { for (const f of big) await get(f); })(), | |
| ]); | |
| report(); | |
| } | |
| /** | |
| * Deletes downloaded files of models that are no longer in the catalog (e.g. after a | |
| * model was dropped from the list), including pieces of unfinished downloads. | |
| * Returns the number of bytes freed. | |
| */ | |
| export async function removeOrphanedModels(knownRepos) { | |
| const known = new Set(knownRepos); | |
| const repoOf = (url) => /^https:\/\/huggingface\.co\/([^/]+\/[^/]+)\/resolve\//.exec(url)?.[1]; | |
| let freed = 0; | |
| for (const name of [CACHE_MODEL, PARTIAL_CACHE]) { | |
| const cache = await caches.open(name); | |
| for (const req of await cache.keys()) { | |
| const repo = repoOf(req.url); | |
| if (!repo || known.has(repo)) continue; | |
| const res = await cache.match(req); | |
| freed += Number(res?.headers.get("Content-Length")) || 0; | |
| await cache.delete(req); | |
| } | |
| } | |
| return freed; | |
| } | |
| /** Deletes one model's downloaded files. */ | |
| export async function removeModel(model) { | |
| const cache = await caches.open(CACHE_MODEL); | |
| const prefix = `https://huggingface.co/${model.repo}/`; | |
| for (const req of await cache.keys()) if (req.url.startsWith(prefix)) await cache.delete(req); | |
| await clearPartial(prefix); // pieces of an unfinished download | |
| } | |
| export async function fetchRemoteManifest() { | |
| const res = await fetch(DISK_BASE + "manifest.json", { cache: "no-store" }); | |
| if (!res.ok) throw new Error(`disk manifest: HTTP ${res.status}`); | |
| return res.json(); | |
| } | |
| // ---------- downloads ---------- | |
| export async function downloadCheerpX(onProgress) { | |
| let done = 0; | |
| for (const f of CX_FILES) { | |
| // Goes through sw.js, which stores it in the CheerpX cache. | |
| const res = await fetch(CX_BASE + f, { mode: "cors" }); | |
| if (!res.ok && res.status !== 204) throw new Error(`CheerpX ${f}: HTTP ${res.status}`); | |
| await res.arrayBuffer(); | |
| onProgress?.(++done / CX_FILES.length); | |
| } | |
| } | |
| /** | |
| * Downloads the chunked disk image. Resumable: chunks already stored are skipped. | |
| * The manifest is written last, so its presence means the image is complete. | |
| */ | |
| export async function downloadDisk(onProgress, { concurrency = 4 } = {}) { | |
| const manifest = await fetchRemoteManifest(); | |
| const cache = await caches.open(CACHE_DISK); | |
| const old = await installedDisk(); | |
| if (old && old.id !== manifest.id) { | |
| await caches.delete(CACHE_DISK); // different image: start clean | |
| } | |
| const disk = await caches.open(CACHE_DISK); | |
| const todo = []; | |
| let doneBytes = 0; | |
| for (let i = 0; i < manifest.chunks.length; i++) { | |
| if (!manifest.chunks[i]) continue; | |
| if (await disk.match(chunkKey(i))) doneBytes += manifest.chunks[i]; | |
| else todo.push(i); | |
| } | |
| const total = manifest.downloadBytes; | |
| onProgress?.(doneBytes, total); | |
| let next = 0; | |
| async function worker() { | |
| while (next < todo.length) { | |
| const i = todo[next++]; | |
| const name = `chunk-${String(i).padStart(5, "0")}.gz`; | |
| let attempt = 0; | |
| for (;;) { | |
| try { | |
| const res = await fetch(DISK_BASE + name, { cache: "no-store" }); | |
| if (!res.ok) throw new Error(`${name}: HTTP ${res.status}`); | |
| // Stored as downloaded (gzipped): the service worker decompresses a chunk when the | |
| // VM first reads it. Keeps Linux at ~0.6 GB of browser storage instead of ~2 GB. | |
| const body = new Uint8Array(await res.arrayBuffer()); | |
| await disk.put(chunkKey(i), new Response(body, { headers: { "Content-Type": "application/octet-stream" } })); | |
| doneBytes += manifest.chunks[i]; | |
| onProgress?.(doneBytes, total); | |
| break; | |
| } catch (err) { | |
| if (++attempt >= 4) throw err; | |
| await new Promise((r) => setTimeout(r, 1000 * attempt)); | |
| } | |
| } | |
| } | |
| } | |
| await Promise.all(Array.from({ length: concurrency }, worker)); | |
| await cache.put(MANIFEST_KEY(), new Response(JSON.stringify(manifest), { headers: { "Content-Type": "application/json" } })); | |
| navigator.serviceWorker.controller?.postMessage({ type: "disk-updated" }); | |
| return manifest; | |
| } | |
| function deleteDb(name) { | |
| return new Promise((resolve) => { | |
| const req = indexedDB.deleteDatabase(name); | |
| req.onsuccess = req.onerror = req.onblocked = () => resolve(); | |
| }); | |
| } | |
| /** Wipes the VM's saved changes. CheerpX names the IndexedDB database "cjFS_/<name>/". */ | |
| export async function resetVmStorage(overlayName) { | |
| const dbs = (await indexedDB.databases?.()) ?? []; | |
| const names = dbs.map((d) => d.name).filter((n) => n?.includes(overlayName)); | |
| for (const n of names.length ? names : ["cjFS_/" + overlayName + "/"]) await deleteDb(n); | |
| } | |
| /** Removes everything this app stored (model, disk, CheerpX, VM files). */ | |
| export async function uninstallAll() { | |
| for (const k of await caches.keys()) { | |
| if (k.startsWith("lwa-") || k === CACHE_MODEL || k.startsWith("transformers")) await caches.delete(k); | |
| } | |
| const dbs = (await indexedDB.databases?.()) ?? []; | |
| for (const db of dbs) if (db.name?.includes("lwa-")) await deleteDb(db.name); | |
| } | |