import { normalizeComboStep } from "@/lib/combos/steps"; import type { SqliteAdapter } from "./adapters/types"; type SqliteDatabase = SqliteAdapter; type JsonRecord = Record; export type DbHealthIssueType = | "integrity_check_failed" | "broken_reference" | "stale_snapshot" | "invalid_state"; export interface DbHealthIssue { type: DbHealthIssueType; table: string; description: string; count: number; } export interface DbHealthCheckResult { isHealthy: boolean; issues: DbHealthIssue[]; repairedCount: number; backupCreated: boolean; autoRepair: boolean; checkedAt: string; } interface RunDbHealthCheckOptions { autoRepair?: boolean; createBackupBeforeRepair?: () => boolean; expectedSchemaVersion?: string; /** * Skip `PRAGMA quick_check` during this run. * Set via env var `OMNIROUTE_SKIP_DB_HEALTHCHECK=1`. * On slow storage (HDD under I/O contention) quick_check can block the * Node.js event loop for minutes. The DB is implicitly validated by * opening it, applying the schema, and running migrations — if corruption * existed, those operations would fail first. */ skipIntegrityCheck?: boolean; } interface ComboRow { id: string; name: string; data: string; sort_order?: number | null; created_at?: string | null; updated_at?: string | null; } interface ComboRepairResult { issueCount: number; repairedCount: number; } interface QuotaSnapshotRow { id?: number; provider?: string | null; connection_id?: string | null; created_at?: string | null; } function isRecord(value: unknown): value is JsonRecord { return !!value && typeof value === "object" && !Array.isArray(value); } function toRecord(value: unknown): JsonRecord { return isRecord(value) ? value : {}; } function toTrimmedString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } function parseJsonRecord(value: string): JsonRecord | null { try { const parsed = JSON.parse(value); return isRecord(parsed) ? parsed : null; } catch { return null; } } function isFiniteNumber(value: unknown): boolean { return typeof value === "number" && Number.isFinite(value); } function hasRows(db: SqliteDatabase, table: string): boolean { const row = db .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") .get(table) as { name?: string } | undefined; return row?.name === table; } function hasProviderConnection(db: SqliteDatabase, connectionId: string): boolean { const row = db .prepare("SELECT 1 AS ok FROM provider_connections WHERE id = ? LIMIT 1") .get(connectionId) as { ok?: number } | undefined; return row?.ok === 1; } function isValidIsoTimestamp(value: unknown): boolean { if (typeof value !== "string" || value.trim().length === 0) return false; return !Number.isNaN(Date.parse(value)); } function buildRepairNote(message: string, checkedAt: string): string { return `[db-health:${checkedAt}] ${message}`; } function buildDisabledCombo(row: ComboRow, checkedAt: string): JsonRecord { const now = checkedAt; return { id: row.id, name: row.name, version: 2, strategy: "priority", models: [], config: {}, isActive: false, isHidden: false, sortOrder: typeof row.sort_order === "number" ? row.sort_order : 0, createdAt: row.created_at || now, updatedAt: now, repairNote: buildRepairNote("Combo payload was rebuilt after invalid JSON was detected.", now), }; } function normalizeComboModels(models: unknown): unknown[] { return Array.isArray(models) ? models : []; } function repairComboRows( db: SqliteDatabase, rows: ComboRow[], checkedAt: string, options: { autoRepair: boolean } ): ComboRepairResult { if (rows.length === 0) return { issueCount: 0, repairedCount: 0 }; const existingComboNames = new Set(rows.map((row) => row.name)); let issueCount = 0; let repairedCount = 0; const updateComboStmt = db.prepare("UPDATE combos SET data = ?, updated_at = ? WHERE id = ?"); for (const row of rows) { const parsed = parseJsonRecord(row.data); if (!parsed) { issueCount += 1; if (options.autoRepair) { const repaired = buildDisabledCombo(row, checkedAt); updateComboStmt.run(JSON.stringify(repaired), checkedAt, row.id); repairedCount += 1; } continue; } const currentModels = normalizeComboModels(parsed.models); if (currentModels.length === 0) continue; const nextModels: unknown[] = []; let removedSteps = 0; let clearedConnectionPins = 0; let normalizedLegacyComboRefs = 0; for (const [index, rawStep] of currentModels.entries()) { if (!isRecord(rawStep)) { if (typeof rawStep === "string") { const normalizedStep = normalizeComboStep(rawStep, { comboName: row.name, index, allCombos: existingComboNames, }); if (normalizedStep?.kind === "combo-ref") { if ( normalizedStep.comboName === row.name || !existingComboNames.has(normalizedStep.comboName) ) { removedSteps += 1; continue; } nextModels.push(normalizedStep); normalizedLegacyComboRefs += 1; continue; } } nextModels.push(rawStep); continue; } if (rawStep.kind === "combo-ref") { const comboName = toTrimmedString(rawStep.comboName); if (!comboName || comboName === row.name || !existingComboNames.has(comboName)) { removedSteps += 1; continue; } nextModels.push(rawStep); continue; } const connectionId = toTrimmedString(rawStep.connectionId); if (connectionId && !hasProviderConnection(db, connectionId)) { const repairedStep = { ...rawStep }; delete repairedStep.connectionId; nextModels.push(repairedStep); clearedConnectionPins += 1; continue; } nextModels.push(rawStep); } if (removedSteps === 0 && clearedConnectionPins === 0 && normalizedLegacyComboRefs === 0) { continue; } issueCount += removedSteps + clearedConnectionPins + normalizedLegacyComboRefs; if (!options.autoRepair) continue; const nextCombo = { ...parsed, models: nextModels, updatedAt: checkedAt, repairNote: buildRepairNote( [ removedSteps > 0 ? `${removedSteps} broken combo step(s) removed.` : null, clearedConnectionPins > 0 ? `${clearedConnectionPins} missing connection pin(s) cleared.` : null, normalizedLegacyComboRefs > 0 ? `${normalizedLegacyComboRefs} legacy combo ref step(s) canonicalized.` : null, ] .filter(Boolean) .join(" "), checkedAt ), ...(nextModels.length === 0 ? { isActive: false } : {}), }; updateComboStmt.run(JSON.stringify(nextCombo), checkedAt, row.id); repairedCount += removedSteps + clearedConnectionPins + normalizedLegacyComboRefs; } return { issueCount, repairedCount }; } function getBrokenQuotaSnapshotRowIds(db: SqliteDatabase): number[] { if (!hasRows(db, "quota_snapshots")) return []; const brokenRowIds = new Set(); const rows = db .prepare("SELECT id, provider, connection_id, created_at FROM quota_snapshots") .all() as QuotaSnapshotRow[]; for (const row of rows) { const connectionId = toTrimmedString(row.connection_id); const missingConnection = !!connectionId && !hasProviderConnection(db, connectionId); const invalidTimestamp = !isValidIsoTimestamp(row.created_at); if ((missingConnection || invalidTimestamp) && typeof row.id === "number") { brokenRowIds.add(row.id); } } return Array.from(brokenRowIds); } function countOrphanQuotaSnapshots(db: SqliteDatabase): number { return getBrokenQuotaSnapshotRowIds(db).length; } function repairQuotaSnapshots(db: SqliteDatabase): number { if (!hasRows(db, "quota_snapshots")) return 0; const brokenRowIds = getBrokenQuotaSnapshotRowIds(db); if (brokenRowIds.length === 0) return 0; const deleteByRowId = db.prepare("DELETE FROM quota_snapshots WHERE id = ?"); let repaired = 0; for (const rowId of brokenRowIds) { repaired += deleteByRowId.run(rowId).changes; } return repaired; } function countOrphanDomainRows( db: SqliteDatabase, table: "domain_budgets" | "domain_cost_history" ) { if (!hasRows(db, table)) return 0; const row = db .prepare( `SELECT COUNT(*) AS count FROM ${table} WHERE api_key_id NOT IN (SELECT id FROM api_keys)` ) .get() as { count?: number } | undefined; return row?.count || 0; } function repairOrphanDomainRows( db: SqliteDatabase, table: "domain_budgets" | "domain_cost_history" ): number { if (!hasRows(db, table)) return 0; return db.prepare(`DELETE FROM ${table} WHERE api_key_id NOT IN (SELECT id FROM api_keys)`).run() .changes; } function countInvalidJsonRows( db: SqliteDatabase, table: "domain_fallback_chains" | "domain_lockout_state" | "domain_circuit_breakers", column: "chain" | "attempts" | "options" ): number { if (!hasRows(db, table)) return 0; const rows = db.prepare(`SELECT ${column} FROM ${table}`).all() as Array>; let invalid = 0; for (const row of rows) { const raw = row[column]; if (raw == null && column === "options") continue; if (typeof raw !== "string") { invalid += 1; continue; } try { JSON.parse(raw); } catch { invalid += 1; } } return invalid; } function repairInvalidJsonRows( db: SqliteDatabase, table: "domain_fallback_chains" | "domain_lockout_state" | "domain_circuit_breakers", column: "chain" | "attempts" | "options" ): number { if (!hasRows(db, table)) return 0; const rows = db.prepare(`SELECT rowid, ${column} FROM ${table}`).all() as Array<{ rowid: number; [key: string]: unknown; }>; const deleteByRowId = db.prepare(`DELETE FROM ${table} WHERE rowid = ?`); const clearOptionsByRowId = db.prepare( "UPDATE domain_circuit_breakers SET options = NULL WHERE rowid = ?" ); let repaired = 0; for (const row of rows) { const raw = row[column]; if (raw == null && table === "domain_circuit_breakers") { continue; } if (typeof raw !== "string") { if (table === "domain_circuit_breakers") { repaired += clearOptionsByRowId.run(row.rowid).changes; continue; } deleteByRowId.run(row.rowid); repaired += 1; continue; } try { JSON.parse(raw); } catch { if (table === "domain_circuit_breakers") { repaired += clearOptionsByRowId.run(row.rowid).changes; continue; } deleteByRowId.run(row.rowid); repaired += 1; } } return repaired; } function getSchemaVersionIssueCount(db: SqliteDatabase, expectedSchemaVersion: string): number { if (!hasRows(db, "db_meta")) return 0; const row = db.prepare("SELECT value FROM db_meta WHERE key = 'schema_version'").get() as | { value?: string | null } | undefined; const current = typeof row?.value === "string" ? row.value : null; return current === expectedSchemaVersion ? 0 : 1; } function repairSchemaVersion(db: SqliteDatabase, expectedSchemaVersion: string): number { if (!hasRows(db, "db_meta")) return 0; return db .prepare("INSERT OR REPLACE INTO db_meta (key, value) VALUES ('schema_version', ?)") .run(expectedSchemaVersion).changes; } export function runDbHealthCheck( db: SqliteDatabase, options: RunDbHealthCheckOptions = {} ): DbHealthCheckResult { const autoRepair = options.autoRepair === true; const expectedSchemaVersion = options.expectedSchemaVersion || "1"; const checkedAt = new Date().toISOString(); const issues: DbHealthIssue[] = []; let repairedCount = 0; let backupCreated = false; let backupAttempted = false; const ensureBackupBeforeRepair = () => { if (!autoRepair || backupAttempted || typeof options.createBackupBeforeRepair !== "function") { return; } backupAttempted = true; backupCreated = options.createBackupBeforeRepair(); }; // Use quick_check instead of integrity_check on startup — integrity_check // does a full page-by-page scan that can take minutes on a fragmented WAL, // causing 7+ minute boot times. quick_check still catches corruption but // skips deep index verification, reducing I/O to seconds. // Skip entirely when skipIntegrityCheck is set (env OMNIROUTE_SKIP_DB_HEALTHCHECK=1). if (!options.skipIntegrityCheck) { const integrityCheck = db.pragma("quick_check") as Array<{ quick_check?: string }>; if (integrityCheck[0]?.quick_check !== "ok") { issues.push({ type: "integrity_check_failed", table: "sqlite", description: "SQLite integrity_check returned a non-ok status.", count: 1, }); } } if (hasRows(db, "combos")) { const comboRows = db .prepare( "SELECT id, name, data, sort_order, created_at, updated_at FROM combos ORDER BY name COLLATE NOCASE ASC" ) .all() as ComboRow[]; const comboRepair = repairComboRows(db, comboRows, checkedAt, { autoRepair }); if (comboRepair.issueCount > 0) { issues.push({ type: "broken_reference", table: "combos", description: "Combos contained broken combo references, legacy combo refs, invalid JSON, or pinned connections that no longer exist.", count: comboRepair.issueCount, }); if (autoRepair) { ensureBackupBeforeRepair(); repairedCount += comboRepair.repairedCount; } } } const orphanQuotaCount = countOrphanQuotaSnapshots(db); if (orphanQuotaCount > 0) { issues.push({ type: "stale_snapshot", table: "quota_snapshots", description: "Quota snapshots referenced missing connections or contained invalid timestamps.", count: orphanQuotaCount, }); if (autoRepair) { ensureBackupBeforeRepair(); repairedCount += repairQuotaSnapshots(db); } } const orphanBudgets = countOrphanDomainRows(db, "domain_budgets"); if (orphanBudgets > 0) { issues.push({ type: "broken_reference", table: "domain_budgets", description: "Domain budgets referenced API keys that no longer exist.", count: orphanBudgets, }); if (autoRepair) { ensureBackupBeforeRepair(); repairedCount += repairOrphanDomainRows(db, "domain_budgets"); } } const orphanCostHistory = countOrphanDomainRows(db, "domain_cost_history"); if (orphanCostHistory > 0) { issues.push({ type: "broken_reference", table: "domain_cost_history", description: "Domain cost history referenced API keys that no longer exist.", count: orphanCostHistory, }); if (autoRepair) { ensureBackupBeforeRepair(); repairedCount += repairOrphanDomainRows(db, "domain_cost_history"); } } const invalidFallbackChains = countInvalidJsonRows(db, "domain_fallback_chains", "chain"); if (invalidFallbackChains > 0) { issues.push({ type: "invalid_state", table: "domain_fallback_chains", description: "Fallback chain rows contained invalid JSON payloads.", count: invalidFallbackChains, }); if (autoRepair) { ensureBackupBeforeRepair(); repairedCount += repairInvalidJsonRows(db, "domain_fallback_chains", "chain"); } } const invalidLockoutState = countInvalidJsonRows(db, "domain_lockout_state", "attempts"); if (invalidLockoutState > 0) { issues.push({ type: "invalid_state", table: "domain_lockout_state", description: "Lockout state rows contained invalid JSON payloads.", count: invalidLockoutState, }); if (autoRepair) { ensureBackupBeforeRepair(); repairedCount += repairInvalidJsonRows(db, "domain_lockout_state", "attempts"); } } const invalidBreakerOptions = countInvalidJsonRows(db, "domain_circuit_breakers", "options"); if (invalidBreakerOptions > 0) { issues.push({ type: "invalid_state", table: "domain_circuit_breakers", description: "Circuit breaker option payloads were invalid JSON.", count: invalidBreakerOptions, }); if (autoRepair) { ensureBackupBeforeRepair(); repairedCount += repairInvalidJsonRows(db, "domain_circuit_breakers", "options"); } } const schemaVersionIssues = getSchemaVersionIssueCount(db, expectedSchemaVersion); if (schemaVersionIssues > 0) { issues.push({ type: "invalid_state", table: "db_meta", description: `db_meta.schema_version did not match expected version ${expectedSchemaVersion}.`, count: schemaVersionIssues, }); if (autoRepair) { ensureBackupBeforeRepair(); repairedCount += repairSchemaVersion(db, expectedSchemaVersion); } } return { isHealthy: issues.length === 0, issues, repairedCount, backupCreated, autoRepair, checkedAt, }; }