MiniCPM5-2B-WebGPU-Pi-HTTP / app /src /local-folder.mjs
Mike0021's picture
Explain binary and oversized files in the editor preview
b74c172 verified
Raw History Blame Contribute Delete
14.9 kB
import { InMemoryFs, MountableFs } from 'just-bash/browser';
// Only enumerate names at connection time. File bytes are loaded on demand,
// into a bounded staging filesystem; only Apply can write to the real handle.
export const LOCAL_FILE_LIMIT = 1024 ** 2;
const TOTAL_LIMIT = 16 * LOCAL_FILE_LIMIT;
const SKIP = new Set(['.git', 'node_modules', '.venv', '__pycache__']);
const same = (a, b) => a?.length === b?.length && (a == null || a.every((v, i) => v === b[i]));
export function encodeBytes(bytes) {
if (bytes == null) return null;
let text = '';
for (let i = 0; i < bytes.length; i += 32768) text += String.fromCharCode(...bytes.subarray(i, i + 32768));
return btoa(text);
}
export const decodeBytes = text => text == null ? null : Uint8Array.from(atob(text), c => c.charCodeAt(0));
export function displayText(bytes) {
if (bytes == null) return '';
let text;
try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); } catch { /* Not UTF-8 text. */ }
if (text === undefined || text.includes('\0')) throw Object.assign(Error('This is a binary or non-UTF-8 file. The editor supports UTF-8 text.'), { code: 'BINARY_FILE' });
return text;
}
const unsupported = () => { throw Error('ENOTSUP: Local folders do not support Unix permissions, timestamps, or links.'); };
export class LocalFolderFs {
constructor(handle) {
this.handle = handle;
this.memory = new InMemoryFs({}, { maxTotalBytes: TOTAL_LIMIT });
this.original = new Map([['/', { kind: 'directory', handle }]]);
this.baselines = new Map();
this.dirty = new Set();
this.operations = new Map();
this.skipped = 0;
this.baselineBytes = 0;
}
static async open(handle, { signal, patches = [] } = {}) {
if (handle?.kind !== 'directory') throw Error('Select a folder.');
const fs = new LocalFolderFs(handle);
let count = 0;
async function walk(dir, prefix, depth) {
if (depth > 40) throw Error('This folder is too deeply nested. Choose a smaller project folder.');
for await (const [name, child] of dir.entries()) {
signal?.throwIfAborted();
if (SKIP.has(name)) { fs.skipped++; continue; }
if (++count > 5000) throw Error('This folder contains more than 5,000 entries. Choose a smaller project folder.');
const path = prefix + '/' + name;
fs.original.set(path, { kind: child.kind, handle: child });
if (child.kind === 'directory') {
await fs.memory.mkdir(path, { recursive: true });
await walk(child, path, depth + 1);
} else fs.memory.writeFileLazy(path, () => fs.loadOriginal(path));
}
}
await walk(handle, '', 0);
for (const patch of patches) {
const path = fs.path(patch.path);
if (path === '/') throw Error('Invalid saved local-folder change.');
const before = decodeBytes(patch.before);
fs.baselineBytes += before?.length ?? 0;
if (fs.baselineBytes > TOTAL_LIMIT) throw Error('Saved changes exceed the 16 MiB local-folder limit.');
fs.baselines.set(path, before);
fs.dirty.add(path);
if (patch.kind === 'directory') await fs.memory.mkdir(path, { recursive: true });
else if (patch.after === null) await fs.memory.rm(path, { force: true });
else {
await fs.memory.mkdir(fs.parent(path), { recursive: true });
await fs.memory.writeFile(path, decodeBytes(patch.after));
}
}
return fs;
}
path(value) {
if (typeof value !== 'string' || value.includes('\0')) throw Error('Invalid file path.');
const path = this.memory.resolvePath('/', value);
if (path.split('/').some(part => SKIP.has(part))) throw Error('This folder is excluded: .git, node_modules, .venv, and __pycache__.');
return path;
}
parent(path) { return path.slice(0, path.lastIndexOf('/')) || '/'; }
async diskBytes(path) {
const handle = await this.diskEntry(path);
if (!handle) return null;
if (handle.kind !== 'file') throw Error('A folder now occupies ' + path + '. No file was overwritten.');
const file = await handle.getFile();
if (file.size > LOCAL_FILE_LIMIT) throw Object.assign(Error(path + ' is larger than the 1 MiB per-file limit.'), { code: 'FILE_TOO_LARGE' });
return new Uint8Array(await file.arrayBuffer());
}
async diskEntry(path) {
let dir = this.handle;
const parts = this.path(path).split('/').filter(Boolean);
if (!parts.length) return dir;
try {
for (const part of parts.slice(0, -1)) dir = await dir.getDirectoryHandle(part);
try { return await dir.getFileHandle(parts.at(-1)); }
catch (e) { if (e.name !== 'TypeMismatchError') throw e; return await dir.getDirectoryHandle(parts.at(-1)); }
} catch (e) { if (e.name === 'NotFoundError') return null; throw e; }
}
async loadOriginal(path) {
const bytes = await this.diskBytes(path);
if (!bytes) throw Error('ENOENT: File was removed from disk: ' + path + '. Refresh the folder.');
// Preserve the version actually read, even if the host file later changes.
if (!this.baselines.has(path)) {
if (this.baselineBytes + bytes.length > TOTAL_LIMIT) throw Error('Local file reads reached the 16 MiB session limit. Apply or discard changes, then Refresh.');
this.baselines.set(path, bytes); this.baselineBytes += bytes.length;
}
return bytes;
}
async baseline(path) {
if (!this.baselines.has(path)) this.baselines.set(path, this.original.get(path)?.kind === 'file' ? await this.loadOriginal(path) : null);
}
async atPath(path, operation) {
// A preview can arrive while an agent tool is working. Do not let a slow
// lazy read materialize its old contents over a more recent staged write.
const previous = this.operations.get(path) ?? Promise.resolve();
const pending = previous.catch(() => {}).then(operation);
this.operations.set(path, pending);
try { return await pending; }
finally { if (this.operations.get(path) === pending) this.operations.delete(path); }
}
async readFile(value, options) { const path = this.path(value); return this.atPath(path, () => this.memory.readFile(path, options)); }
async readFileBuffer(value) { const path = this.path(value); return this.atPath(path, () => this.memory.readFileBuffer(path)); }
async readFileBytes(value) { const path = this.path(value); return this.atPath(path, () => this.memory.readFileBytes(path)); }
async exists(path) { return this.memory.exists(this.path(path)); }
async stat(value) {
const path = this.path(value), entry = this.original.get(path);
// InMemoryFs.stat materializes lazy files. Avoid that for ls/find/indexing.
if (entry?.kind === 'file' && !this.dirty.has(path)) {
const file = await entry.handle.getFile();
return { isFile: true, isDirectory: false, isSymbolicLink: false, mode: 0o644, size: this.baselines.get(path)?.length ?? file.size, mtime: new Date(file.lastModified) };
}
return this.memory.stat(path);
}
async lstat(path) { return this.stat(path); }
async realpath(path) { return this.memory.realpath(this.path(path)); }
resolvePath(base, path) { return this.memory.resolvePath(base, path); }
getAllPaths() { return this.memory.getAllPaths(); }
async readdir(path) { return this.memory.readdir(this.path(path)); }
async readdirWithFileTypes(path) { return this.memory.readdirWithFileTypes(this.path(path)); }
async mkdir(value, options) {
const path = this.path(value), previous = new Set(this.getAllPaths());
await this.memory.mkdir(path, options);
for (const created of this.getAllPaths()) if (!previous.has(created)) { this.dirty.add(created); this.baselines.set(created, null); }
}
async writeFile(value, content, options) {
const path = this.path(value);
return this.atPath(path, () => this.writeAtPath(path, content, options));
}
async writeAtPath(path, content, options) {
if (path === '/') throw Error('EISDIR: Cannot replace the selected folder.');
// Decode using just-bash's encoding semantics before enforcing byte limits.
const temporary = new InMemoryFs({}, { maxTotalBytes: LOCAL_FILE_LIMIT });
await temporary.writeFile('/data', content, options);
const bytes = await temporary.readFileBuffer('/data');
if (bytes.length > LOCAL_FILE_LIMIT) throw Error('Local files are limited to 1 MiB.');
if (!await this.memory.exists(this.parent(path))) throw Error('ENOENT: Parent folder does not exist: ' + this.parent(path));
await this.baseline(path);
await this.memory.writeFile(path, bytes);
this.dirty.add(path);
}
async appendFile(path, content, options) {
const old = await this.exists(path) ? await this.readFileBuffer(path) : new Uint8Array();
const temporary = new InMemoryFs({ '/data': old }, { maxTotalBytes: LOCAL_FILE_LIMIT });
await temporary.appendFile('/data', content, options);
await this.writeFile(path, await temporary.readFileBuffer('/data'));
}
async rm(value, options) {
const path = this.path(value);
return this.atPath(path, () => this.removeAtPath(path, options));
}
async removeAtPath(path, options) {
if (path === '/') throw Error('Cannot remove the selected local folder.');
if (!await this.exists(path)) return this.memory.rm(path, options);
if ((await this.stat(path)).isDirectory) {
if (this.original.has(path)) throw Error('Removing or renaming existing local folders is not supported. Remove individual files instead.');
for (const p of this.getAllPaths().filter(p => p.startsWith(path + '/'))) {
if (this.original.has(p)) throw Error('This folder contains existing local files.');
}
const removed = this.getAllPaths().filter(p => p === path || p.startsWith(path + '/'));
await this.memory.rm(path, options);
for (const p of removed) { this.dirty.delete(p); this.baselines.delete(p); }
} else {
await this.baseline(path); await this.memory.rm(path, options); this.dirty.add(path);
}
}
async cp(source, destination, options) {
const src = this.path(source); let dest = this.path(destination);
if (await this.exists(dest) && (await this.stat(dest)).isDirectory) dest += '/' + src.split('/').at(-1);
if (src === dest || dest.startsWith(src + '/')) throw Error('Cannot copy a path into itself.');
if ((await this.stat(src)).isDirectory) {
if (!options?.recursive) throw Error('EISDIR: Use cp -r to copy a directory.');
await this.mkdir(dest, { recursive: true });
for (const name of await this.readdir(src)) await this.cp(src + '/' + name, dest + '/' + name, options);
} else await this.writeFile(dest, await this.readFileBuffer(src));
}
async mv(src, dest) {
if ((await this.stat(src)).isDirectory) throw Error('Renaming local folders is not supported.');
await this.cp(src, dest); await this.rm(src);
}
chmod = unsupported;
symlink = unsupported;
link = unsupported;
readlink = unsupported;
utimes = unsupported;
async patches() {
const patches = [];
for (const path of [...this.dirty].sort()) {
const before = this.baselines.get(path) ?? null;
const exists = await this.exists(path);
const kind = exists && (await this.memory.stat(path)).isDirectory ? 'directory' : 'file';
const after = exists && kind === 'file' ? await this.readFileBuffer(path) : null;
if (kind === 'file' && same(before, after)) continue;
patches.push({ path, kind, before: encodeBytes(before), after: encodeBytes(after) });
}
return patches;
}
async listing() {
const files = Object.create(null);
for (const path of this.getAllPaths()) {
const entry = this.original.get(path);
if (entry?.kind === 'file' || (!entry && (await this.memory.stat(path)).isFile)) files['/workspace' + path] = null;
}
return files;
}
async status() {
return { name: this.handle.name, count: (await this.patches()).length, skipped: this.skipped };
}
async apply(signal) {
const patches = await this.patches();
// Preflight the entire set before touching disk. Check again per file to
// catch edits made during application. File System Access has no multi-file transaction.
const check = async patch => {
const actual = await this.diskEntry(patch.path);
if (patch.kind === 'directory') {
if (actual) throw Error('Conflict: ' + patch.path + ' was created on disk.');
} else if (!same(await this.diskBytes(patch.path), decodeBytes(patch.before))) {
throw Error('Conflict: ' + patch.path + ' changed on disk. Nothing will overwrite it. Discard staged edits and refresh to read the new version.');
}
};
for (const patch of patches) { signal?.throwIfAborted(); await check(patch); }
let applied = 0;
try {
for (const patch of patches) {
signal?.throwIfAborted(); await check(patch);
const parts = patch.path.split('/').filter(Boolean); let dir = this.handle;
for (const part of parts.slice(0, -1)) dir = await dir.getDirectoryHandle(part);
const name = parts.at(-1);
if (patch.kind === 'directory') {
const handle = await dir.getDirectoryHandle(name, { create: true });
this.original.set(patch.path, { kind: 'directory', handle });
} else if (patch.after === null) {
await dir.removeEntry(name); this.original.delete(patch.path);
this.baselineBytes -= this.baselines.get(patch.path)?.length ?? 0; this.baselines.delete(patch.path);
} else {
const handle = await dir.getFileHandle(name, { create: true });
let stream;
try {
stream = await handle.createWritable(); await stream.write(decodeBytes(patch.after));
signal?.throwIfAborted(); await stream.close();
} catch (e) {
await stream?.abort().catch(() => {});
// create:true creates an empty entry before createWritable runs.
// If writing failed, remove only our still-empty new entry.
if (patch.before === null) {
try { if ((await handle.getFile()).size === 0 && await handle.isSameEntry(await dir.getFileHandle(name))) await dir.removeEntry(name); } catch {}
}
throw e;
}
this.original.set(patch.path, { kind: 'file', handle });
this.baselineBytes += decodeBytes(patch.after).length - (this.baselines.get(patch.path)?.length ?? 0);
this.baselines.set(patch.path, decodeBytes(patch.after));
}
this.dirty.delete(patch.path); applied++;
}
} catch (error) {
throw Error(`${applied} of ${patches.length} changes applied. Remaining changes are still staged. ${error.message}`);
}
return applied;
}
mount() {
return new MountableFs({ mounts: [{ mountPoint: '/workspace', filesystem: this }] });
}
}