/** * Database cleanup functions for removing old data based on retention policies. * * @module lib/db/cleanup */ import { getDbInstance } from "./core"; import { getUserDatabaseSettings } from "./databaseSettings"; import { rollupUsageHistoryBeforeDate } from "@/lib/usage/aggregateHistory"; import { purgeCallLogArtifactDirectory } from "@/lib/usage/callLogArtifacts"; interface CleanupResult { deleted: number; deletedArtifacts?: number; errors: number; } function getRetentionSettings() { return getUserDatabaseSettings().retention; } /** * Clean up old quota_snapshots based on retention settings. */ export async function cleanupQuotaSnapshots(): Promise { const db = getDbInstance(); const retention = getRetentionSettings(); const retentionDays = retention.quotaSnapshots; const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - retentionDays); const cutoffISO = cutoffDate.toISOString(); const result: CleanupResult = { deleted: 0, errors: 0 }; try { const stmt = db.prepare("DELETE FROM quota_snapshots WHERE created_at < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; console.log( `[Cleanup] Deleted ${result.deleted} quota_snapshots older than ${retentionDays} days` ); } catch (err: unknown) { console.error("[Cleanup] Error cleaning quota_snapshots:", err); result.errors++; } return result; } /** * Clean up old call_logs based on retention settings. */ export async function cleanupCallLogs(): Promise { const db = getDbInstance(); const retention = getRetentionSettings(); const retentionDays = retention.callLogs; const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - retentionDays); const cutoffISO = cutoffDate.toISOString(); const result: CleanupResult = { deleted: 0, errors: 0 }; try { const stmt = db.prepare("DELETE FROM call_logs WHERE timestamp < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; console.log(`[Cleanup] Deleted ${result.deleted} call_logs older than ${retentionDays} days`); } catch (err: unknown) { console.error("[Cleanup] Error cleaning call_logs:", err); result.errors++; } return result; } /** * Clean up old usage_history based on retention settings. */ export async function cleanupUsageHistory(): Promise { const db = getDbInstance(); const retention = getRetentionSettings(); const retentionDays = retention.usageHistory; const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - retentionDays); const cutoffISO = cutoffDate.toISOString(); const cutoffDateStr = cutoffISO.split("T")[0]; const result: CleanupResult = { deleted: 0, errors: 0 }; // Roll up rows that are about to be deleted into daily_usage_summary so that the // analytics route can still surface historical data via the UNION query. The rollup // uses the exact same day boundary as the DELETE below, so every deleted row // is guaranteed to have been aggregated first. // // rollupUsageHistoryBeforeDate catches its own errors and reports them via the // returned result, so we inspect that rather than relying on a thrown exception. // If the rollup failed, abort the DELETE to avoid permanently losing raw usage data // that was never aggregated. const rollupResult = await rollupUsageHistoryBeforeDate(cutoffDateStr); if (rollupResult.errors > 0) { console.error( "[Cleanup] Aborting usage_history deletion because the pre-delete rollup failed." ); result.errors += rollupResult.errors; return result; } try { const stmt = db.prepare("DELETE FROM usage_history WHERE timestamp < ?"); const runResult = stmt.run(cutoffDateStr); result.deleted = runResult.changes; console.log( `[Cleanup] Deleted ${result.deleted} usage_history older than ${retentionDays} days` ); } catch (err: unknown) { console.error("[Cleanup] Error cleaning usage_history:", err); result.errors++; } return result; } /** * Clean up old compression_analytics based on retention settings. */ export async function cleanupCompressionAnalytics(): Promise { const db = getDbInstance(); const retention = getRetentionSettings(); const retentionDays = retention.compressionAnalytics; const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - retentionDays); const cutoffISO = cutoffDate.toISOString(); const result: CleanupResult = { deleted: 0, errors: 0 }; try { const stmt = db.prepare("DELETE FROM compression_analytics WHERE timestamp < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; console.log( `[Cleanup] Deleted ${result.deleted} compression_analytics older than ${retentionDays} days` ); } catch (err: unknown) { console.error("[Cleanup] Error cleaning compression_analytics:", err); result.errors++; } return result; } /** * Clean up old mcp_audit_log based on retention settings. */ export async function cleanupMcpAudit(): Promise { const db = getDbInstance(); const retention = getRetentionSettings(); const retentionDays = retention.mcpAudit; const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - retentionDays); const cutoffISO = cutoffDate.toISOString(); const result: CleanupResult = { deleted: 0, errors: 0 }; try { const stmt = db.prepare("DELETE FROM mcp_tool_audit WHERE timestamp < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; console.log( `[Cleanup] Deleted ${result.deleted} mcp_audit_log older than ${retentionDays} days` ); } catch (err: unknown) { console.error("[Cleanup] Error cleaning mcp_audit_log:", err); result.errors++; } return result; } /** * Clean up old a2a_events based on retention settings. */ export async function cleanupA2aEvents(): Promise { const db = getDbInstance(); const retention = getRetentionSettings(); const retentionDays = retention.a2aEvents; const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - retentionDays); const cutoffISO = cutoffDate.toISOString(); const result: CleanupResult = { deleted: 0, errors: 0 }; try { const stmt = db.prepare("DELETE FROM a2a_task_events WHERE timestamp < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; console.log(`[Cleanup] Deleted ${result.deleted} a2a_events older than ${retentionDays} days`); } catch (err: unknown) { console.error("[Cleanup] Error cleaning a2a_events:", err); result.errors++; } return result; } /** * Clean up old memory_entries based on retention settings. */ export async function cleanupMemoryEntries(): Promise { const db = getDbInstance(); const retention = getRetentionSettings(); const retentionDays = retention.memoryEntries; const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - retentionDays); const cutoffISO = cutoffDate.toISOString(); const result: CleanupResult = { deleted: 0, errors: 0 }; try { const stmt = db.prepare("DELETE FROM memories WHERE created_at < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; console.log( `[Cleanup] Deleted ${result.deleted} memory_entries older than ${retentionDays} days` ); } catch (err: unknown) { console.error("[Cleanup] Error cleaning memory_entries:", err); result.errors++; } return result; } /** * Run all cleanup functions if auto-cleanup is enabled. */ export async function runAutoCleanup(): Promise<{ totalDeleted: number; totalErrors: number; results: Record; }> { const retention = getRetentionSettings(); const autoCleanupEnabled = retention.autoCleanupEnabled; if (!autoCleanupEnabled) { console.log("[Cleanup] Auto-cleanup is disabled"); return { totalDeleted: 0, totalErrors: 0, results: {} }; } console.log("[Cleanup] Starting auto-cleanup..."); const results: Record = { quotaSnapshots: await cleanupQuotaSnapshots(), callLogs: await cleanupCallLogs(), usageHistory: await cleanupUsageHistory(), compressionAnalytics: await cleanupCompressionAnalytics(), mcpAudit: await cleanupMcpAudit(), a2aEvents: await cleanupA2aEvents(), memoryEntries: await cleanupMemoryEntries(), proxyLogs: await cleanupProxyLogs(), }; const totalDeleted = Object.values(results).reduce((sum, r) => sum + r.deleted, 0); const totalErrors = Object.values(results).reduce((sum, r) => sum + r.errors, 0); console.log(`[Cleanup] Auto-cleanup complete: ${totalDeleted} deleted, ${totalErrors} errors`); return { totalDeleted, totalErrors, results }; } /** * Purge ALL quota_snapshots immediately (no retention check). */ export async function purgeQuotaSnapshots(): Promise { const db = getDbInstance(); const result: CleanupResult = { deleted: 0, errors: 0 }; try { const stmt = db.prepare("DELETE FROM quota_snapshots"); const runResult = stmt.run(); result.deleted = runResult.changes; console.log(`[Cleanup] Purged ${result.deleted} quota_snapshots`); } catch (err: unknown) { console.error("[Cleanup] Error purging quota_snapshots:", err); result.errors++; } return result; } /** * Purge ALL call_logs immediately (no retention check). */ export async function purgeCallLogs(): Promise { const db = getDbInstance(); const result: CleanupResult = { deleted: 0, deletedArtifacts: 0, errors: 0 }; try { const runResult = db.prepare("DELETE FROM call_logs").run(); result.deleted = runResult.changes; console.log(`[Cleanup] Purged ${result.deleted} call_logs`); } catch (err: unknown) { console.error("[Cleanup] Error purging call_logs:", err); result.errors++; } const artifactResult = purgeCallLogArtifactDirectory(); result.deletedArtifacts = artifactResult.deletedArtifacts; result.errors += artifactResult.errors; if (artifactResult.errors === 0) { console.log(`[Cleanup] Purged ${result.deletedArtifacts} call log artifact(s)`); } return result; } /** * Purge ALL request_detail_logs immediately (no retention check). */ export async function purgeDetailedLogs(): Promise { const db = getDbInstance(); const result: CleanupResult = { deleted: 0, errors: 0 }; try { const stmt = db.prepare("DELETE FROM request_detail_logs"); const runResult = stmt.run(); result.deleted = runResult.changes; console.log(`[Cleanup] Purged ${result.deleted} request_detail_logs`); } catch (err: unknown) { console.error("[Cleanup] Error purging request_detail_logs:", err); result.errors++; } return result; } /** * Clean up old proxy_logs based on retention settings. * Uses the same retention period as call_logs (30 days default). */ export async function cleanupProxyLogs(): Promise { const db = getDbInstance(); const retention = getRetentionSettings(); const retentionDays = retention.callLogs; const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - retentionDays); const cutoffISO = cutoffDate.toISOString(); const result: CleanupResult = { deleted: 0, errors: 0 }; try { const stmt = db.prepare("DELETE FROM proxy_logs WHERE timestamp < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; console.log( `[Cleanup] Deleted ${result.deleted} proxy_logs older than ${retentionDays} days` ); } catch (err: unknown) { console.error("[Cleanup] Error cleaning proxy_logs:", err); result.errors++; } return result; } // ──────────────── Background Cleanup Scheduler ──────────────── const CLEANUP_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours let _cleanupSchedulerTimer: ReturnType | null = null; /** * Start the background cleanup scheduler. Runs cleanup on startup * and then every 6 hours. Runs VACUUM after deletes to reclaim disk space. * * Without this, tables grow unboundedly (compression_analytics 600K+ rows, * usage_history 250K+ rows) causing 1.4GB+ SQLite files and 3-8GB RSS * from better-sqlite3 memory mapping. */ export function startCleanupScheduler(): void { if (_cleanupSchedulerTimer) return; // Run cleanup 30s after startup (let the server initialize first). setTimeout(async () => { try { const result = await runAutoCleanup(); const proxyResult = await cleanupProxyLogs(); const totalDeleted = result.totalDeleted + proxyResult.deleted; if (totalDeleted > 0) { console.log( `[Cleanup] Startup cleanup freed ${totalDeleted} rows. Running VACUUM...` ); try { const db = getDbInstance(); db.exec("VACUUM"); console.log("[Cleanup] VACUUM completed after startup cleanup."); } catch (vacErr) { console.error("[Cleanup] VACUUM after cleanup failed:", vacErr); } } } catch (err) { console.error("[Cleanup] Startup cleanup failed:", err); } }, 30_000); // Schedule periodic cleanup every 6 hours. _cleanupSchedulerTimer = setInterval(async () => { try { const result = await runAutoCleanup(); const proxyResult = await cleanupProxyLogs(); const totalDeleted = result.totalDeleted + proxyResult.deleted; if (totalDeleted > 0) { console.log( `[Cleanup] Periodic cleanup freed ${totalDeleted} rows. Running VACUUM...` ); try { const db = getDbInstance(); db.exec("VACUUM"); console.log("[Cleanup] VACUUM completed after periodic cleanup."); } catch (vacErr) { console.error("[Cleanup] VACUUM after cleanup failed:", vacErr); } } } catch (err) { console.error("[Cleanup] Periodic cleanup failed:", err); } }, CLEANUP_INTERVAL_MS); // Don't keep the process alive solely for cleanup. if (_cleanupSchedulerTimer && typeof _cleanupSchedulerTimer.unref === "function") { _cleanupSchedulerTimer.unref(); } console.log("[Cleanup] Background cleanup scheduler started (every 6 hours)."); } /** * Stop the background cleanup scheduler (for tests). */ export function stopCleanupScheduler(): void { if (_cleanupSchedulerTimer) { clearInterval(_cleanupSchedulerTimer); _cleanupSchedulerTimer = null; } }