Spaces:
Paused
Paused
File size: 3,402 Bytes
fb4d8fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | import type { DatabaseSync } from "node:sqlite";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { buildFileEntry, listMemoryFiles, type MemoryFileEntry } from "./internal.js";
const log = createSubsystemLogger("memory");
type ProgressState = {
completed: number;
total: number;
label?: string;
report: (update: { completed: number; total: number; label?: string }) => void;
};
export async function syncMemoryFiles(params: {
workspaceDir: string;
extraPaths?: string[];
db: DatabaseSync;
needsFullReindex: boolean;
progress?: ProgressState;
batchEnabled: boolean;
concurrency: number;
runWithConcurrency: <T>(tasks: Array<() => Promise<T>>, concurrency: number) => Promise<T[]>;
indexFile: (entry: MemoryFileEntry) => Promise<void>;
vectorTable: string;
ftsTable: string;
ftsEnabled: boolean;
ftsAvailable: boolean;
model: string;
}) {
const files = await listMemoryFiles(params.workspaceDir, params.extraPaths);
const fileEntries = await Promise.all(
files.map(async (file) => buildFileEntry(file, params.workspaceDir)),
);
log.debug("memory sync: indexing memory files", {
files: fileEntries.length,
needsFullReindex: params.needsFullReindex,
batch: params.batchEnabled,
concurrency: params.concurrency,
});
const activePaths = new Set(fileEntries.map((entry) => entry.path));
if (params.progress) {
params.progress.total += fileEntries.length;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
label: params.batchEnabled ? "Indexing memory files (batch)..." : "Indexing memory files…",
});
}
const tasks = fileEntries.map((entry) => async () => {
const record = params.db
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
.get(entry.path, "memory") as { hash: string } | undefined;
if (!params.needsFullReindex && record?.hash === entry.hash) {
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
return;
}
await params.indexFile(entry);
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
});
await params.runWithConcurrency(tasks, params.concurrency);
const staleRows = params.db
.prepare(`SELECT path FROM files WHERE source = ?`)
.all("memory") as Array<{ path: string }>;
for (const stale of staleRows) {
if (activePaths.has(stale.path)) {
continue;
}
params.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
try {
params.db
.prepare(
`DELETE FROM ${params.vectorTable} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`,
)
.run(stale.path, "memory");
} catch {}
params.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory");
if (params.ftsEnabled && params.ftsAvailable) {
try {
params.db
.prepare(`DELETE FROM ${params.ftsTable} WHERE path = ? AND source = ? AND model = ?`)
.run(stale.path, "memory", params.model);
} catch {}
}
}
}
|