/** * @license * Copyright 2013 The Emscripten Authors * SPDX-License-Identifier: MIT */ addToLibrary({ $IDBFS__deps: ['$FS', '$MEMFS', '$PATH'], $IDBFS__postset: () => addAtExit('IDBFS.quit();'), $IDBFS: { dbs: {}, indexedDB: () => { #if ASSERTIONS assert(typeof indexedDB != 'undefined', 'IDBFS used, but indexedDB not supported'); #endif return indexedDB; }, DB_VERSION: 21, DB_STORE_NAME: 'FILE_DATA', // Queues a new VFS -> IDBFS synchronization operation queuePersist: (mount) => { function onPersistComplete() { if (mount.idbPersistState === 'again') startPersist(); // If a new sync request has appeared in between, kick off a new sync else mount.idbPersistState = 0; // Otherwise reset sync state back to idle to wait for a new sync later } function startPersist() { mount.idbPersistState = 'idb'; // Mark that we are currently running a sync operation IDBFS.syncfs(mount, /*populate:*/false, onPersistComplete); } if (!mount.idbPersistState) { // Programs typically write/copy/move multiple files in the in-memory // filesystem within a single app frame, so when a filesystem sync // command is triggered, do not start it immediately, but only after // the current frame is finished. This way all the modified files // inside the main loop tick will be batched up to the same sync. mount.idbPersistState = setTimeout(startPersist, 0); } else if (mount.idbPersistState === 'idb') { // There is an active IndexedDB sync operation in-flight, but we now // have accumulated more files to sync. We should therefore queue up // a new sync after the current one finishes so that all writes // will be properly persisted. mount.idbPersistState = 'again'; } }, mount: (mount) => { // reuse core MEMFS functionality var mnt = MEMFS.mount(mount); // If the automatic IDBFS persistence option has been selected, then automatically persist // all modifications to the filesystem as they occur. if (mount?.opts?.autoPersist) { mount.idbPersistState = 0; // IndexedDB sync starts in idle state var memfs_node_ops = mnt.node_ops; mnt.node_ops = {...mnt.node_ops}; // Clone node_ops to inject write tracking mnt.node_ops.mknod = (parent, name, mode, dev) => { var node = memfs_node_ops.mknod(parent, name, mode, dev); // Propagate injected node_ops to the newly created child node node.node_ops = mnt.node_ops; // Remember for each IDBFS node which IDBFS mount point they came from so we know which mount to persist on modification. node.idbfs_mount = mnt.mount; // Remember original MEMFS stream_ops for this node node.memfs_stream_ops = node.stream_ops; // Clone stream_ops to inject write tracking node.stream_ops = {...node.stream_ops}; // Track all file writes node.stream_ops.write = (stream, buffer, offset, length, position, canOwn) => { // This file has been modified, we must persist IndexedDB when this file closes stream.node.isModified = true; return node.memfs_stream_ops.write(stream, buffer, offset, length, position, canOwn); }; // Persist IndexedDB on file close node.stream_ops.close = (stream) => { var n = stream.node; if (n.isModified) { IDBFS.queuePersist(n.idbfs_mount); n.isModified = false; } if (n.memfs_stream_ops.close) return n.memfs_stream_ops.close(stream); }; // Persist the node we just created to IndexedDB IDBFS.queuePersist(mnt.mount); return node; }; // Also kick off persisting the filesystem on other operations that modify the filesystem. mnt.node_ops.rmdir = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.rmdir(...args)); mnt.node_ops.symlink = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.symlink(...args)); mnt.node_ops.unlink = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.unlink(...args)); mnt.node_ops.rename = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.rename(...args)); } return mnt; }, syncfs: (mount, populate, callback) => { IDBFS.getLocalSet(mount, (err, local) => { if (err) return callback(err); IDBFS.getRemoteSet(mount, (err, remote) => { if (err) return callback(err); var src = populate ? remote : local; var dst = populate ? local : remote; IDBFS.reconcile(src, dst, callback); }); }); }, quit: () => { for (var value of Object.values(IDBFS.dbs)) { value.close() } IDBFS.dbs = {}; }, getDB: (name, callback) => { // check the cache first var db = IDBFS.dbs[name]; if (db) { return callback(null, db); } var req; try { req = IDBFS.indexedDB().open(name, IDBFS.DB_VERSION); } catch (e) { return callback(e); } if (!req) { return callback("Unable to connect to IndexedDB"); } req.onupgradeneeded = (e) => { var db = /** @type {IDBDatabase} */ (e.target.result); var transaction = e.target.transaction; var fileStore; if (db.objectStoreNames.contains(IDBFS.DB_STORE_NAME)) { fileStore = transaction.objectStore(IDBFS.DB_STORE_NAME); } else { fileStore = db.createObjectStore(IDBFS.DB_STORE_NAME); } if (!fileStore.indexNames.contains('timestamp')) { fileStore.createIndex('timestamp', 'timestamp', { unique: false }); } }; req.onsuccess = () => { db = /** @type {IDBDatabase} */ (req.result); // add to the cache IDBFS.dbs[name] = db; callback(null, db); }; req.onerror = (e) => { callback(e.target.error); e.preventDefault(); }; }, getLocalSet: (mount, callback) => { var entries = {}; function isRealDir(p) { return p !== '.' && p !== '..'; }; function toAbsolute(root) { return (p) => PATH.join2(root, p); }; var check = FS.readdir(mount.mountpoint).filter(isRealDir).map(toAbsolute(mount.mountpoint)); while (check.length) { var path = check.pop(); var stat; try { stat = FS.lstat(path); } catch (e) { return callback(e); } if (FS.isDir(stat.mode)) { check.push(...FS.readdir(path).filter(isRealDir).map(toAbsolute(path))); } entries[path] = { 'timestamp': stat.mtime }; } return callback(null, { type: 'local', entries: entries }); }, getRemoteSet: (mount, callback) => { var entries = {}; IDBFS.getDB(mount.mountpoint, (err, db) => { if (err) return callback(err); try { var transaction = db.transaction([IDBFS.DB_STORE_NAME], 'readonly'); transaction.onerror = (e) => { callback(e.target.error); e.preventDefault(); }; var store = transaction.objectStore(IDBFS.DB_STORE_NAME); var index = store.index('timestamp'); index.openKeyCursor().onsuccess = (event) => { var cursor = event.target.result; if (!cursor) { return callback(null, { type: 'remote', db, entries }); } entries[cursor.primaryKey] = { 'timestamp': cursor.key }; cursor.continue(); }; } catch (e) { return callback(e); } }); }, loadLocalEntry: (path, callback) => { var stat, node; try { var lookup = FS.lookupPath(path); node = lookup.node; stat = FS.lstat(path); } catch (e) { return callback(e); } if (FS.isDir(stat.mode)) { return callback(null, { 'timestamp': stat.mtime, 'mode': stat.mode }); } else if (FS.isLink(stat.mode)) { return callback(null, { 'timestamp': stat.mtime, 'mode': stat.mode, 'link': node.link, }); } else if (FS.isFile(stat.mode)) { // Performance consideration: storing a normal JavaScript array to a IndexedDB is much slower than storing a typed array. // Therefore always convert the file contents to a typed array first before writing the data to IndexedDB. node.contents = MEMFS.getFileDataAsTypedArray(node); return callback(null, { 'timestamp': stat.mtime, 'mode': stat.mode, 'contents': node.contents }); } else { return callback(new Error('node type not supported')); } }, storeLocalEntry: (path, entry, callback) => { try { if (FS.isDir(entry['mode'])) { FS.mkdirTree(path, entry['mode']); } else if (FS.isLink(entry['mode'])) { FS.symlink(entry['link'], path); } else if (FS.isFile(entry['mode'])) { FS.writeFile(path, entry['contents'], { canOwn: true }); } else { return callback(new Error('node type not supported')); } FS.chmod(path, entry['mode']); FS.utime(path, entry['timestamp'], entry['timestamp']); } catch (e) { return callback(e); } callback(null); }, removeLocalEntry: (path, callback) => { try { var stat = FS.lstat(path); if (FS.isDir(stat.mode)) { FS.rmdir(path); } else { FS.unlink(path); } } catch (e) { return callback(e); } callback(null); }, loadRemoteEntry: (store, path, callback) => { var req = store.get(path); req.onsuccess = (event) => callback(null, event.target.result); req.onerror = (e) => { callback(e.target.error); e.preventDefault(); }; }, storeRemoteEntry: (store, path, entry, callback) => { try { var req = store.put(entry, path); } catch (e) { callback(e); return; } req.onsuccess = (event) => callback(); req.onerror = (e) => { callback(e.target.error); e.preventDefault(); }; }, removeRemoteEntry: (store, path, callback) => { var req = store.delete(path); req.onsuccess = (event) => callback(); req.onerror = (e) => { callback(e.target.error); e.preventDefault(); }; }, reconcile: (src, dst, callback) => { var total = 0; var create = []; for (var [key, e] of Object.entries(src.entries)) { var e2 = dst.entries[key]; if (!e2 || e['timestamp'].getTime() != e2['timestamp'].getTime()) { create.push(key); total++; } } var remove = []; for (var key of Object.keys(dst.entries)) { if (!src.entries[key]) { remove.push(key); total++; } } if (!total) { return callback(null); } var errored = false; var db = src.type === 'remote' ? src.db : dst.db; var transaction = db.transaction([IDBFS.DB_STORE_NAME], 'readwrite'); var store = transaction.objectStore(IDBFS.DB_STORE_NAME); function done(err) { if (err && !errored) { errored = true; return callback(err); } }; // transaction may abort if (for example) there is a QuotaExceededError transaction.onerror = transaction.onabort = (e) => { done(e.target.error); e.preventDefault(); }; transaction.oncomplete = (e) => { if (!errored) { callback(null); } }; // sort paths in ascending order so directory entries are created // before the files inside them for (const path of create.sort()) { if (dst.type === 'local') { IDBFS.loadRemoteEntry(store, path, (err, entry) => { if (err) return done(err); IDBFS.storeLocalEntry(path, entry, done); }); } else { IDBFS.loadLocalEntry(path, (err, entry) => { if (err) return done(err); IDBFS.storeRemoteEntry(store, path, entry, done); }); } } // sort paths in descending order so files are deleted before their // parent directories for (var path of remove.sort().reverse()) { if (dst.type === 'local') { IDBFS.removeLocalEntry(path, done); } else { IDBFS.removeRemoteEntry(store, path, done); } } } } }); if (WASMFS) { error("using -lidbfs is not currently supported in WasmFS."); }