ai_api / src /lib /db /healthCheck.ts
Yogesh
initial deploy
cd8bd0a
Raw
History Blame Contribute Delete
17.3 kB
import { normalizeComboStep } from "@/lib/combos/steps";
import type { SqliteAdapter } from "./adapters/types";
type SqliteDatabase = SqliteAdapter;
type JsonRecord = Record<string, unknown>;
export type DbHealthIssueType =
| "integrity_check_failed"
| "broken_reference"
| "stale_snapshot"
| "invalid_state";
export interface DbHealthIssue {
type: DbHealthIssueType;
table: string;
description: string;
count: number;
}
export interface DbHealthCheckResult {
isHealthy: boolean;
issues: DbHealthIssue[];
repairedCount: number;
backupCreated: boolean;
autoRepair: boolean;
checkedAt: string;
}
interface RunDbHealthCheckOptions {
autoRepair?: boolean;
createBackupBeforeRepair?: () => boolean;
expectedSchemaVersion?: string;
/**
* Skip `PRAGMA quick_check` during this run.
* Set via env var `OMNIROUTE_SKIP_DB_HEALTHCHECK=1`.
* On slow storage (HDD under I/O contention) quick_check can block the
* Node.js event loop for minutes. The DB is implicitly validated by
* opening it, applying the schema, and running migrations — if corruption
* existed, those operations would fail first.
*/
skipIntegrityCheck?: boolean;
}
interface ComboRow {
id: string;
name: string;
data: string;
sort_order?: number | null;
created_at?: string | null;
updated_at?: string | null;
}
interface ComboRepairResult {
issueCount: number;
repairedCount: number;
}
interface QuotaSnapshotRow {
id?: number;
provider?: string | null;
connection_id?: string | null;
created_at?: string | null;
}
function isRecord(value: unknown): value is JsonRecord {
return !!value && typeof value === "object" && !Array.isArray(value);
}
function toRecord(value: unknown): JsonRecord {
return isRecord(value) ? value : {};
}
function toTrimmedString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function parseJsonRecord(value: string): JsonRecord | null {
try {
const parsed = JSON.parse(value);
return isRecord(parsed) ? parsed : null;
} catch {
return null;
}
}
function isFiniteNumber(value: unknown): boolean {
return typeof value === "number" && Number.isFinite(value);
}
function hasRows(db: SqliteDatabase, table: string): boolean {
const row = db
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?")
.get(table) as { name?: string } | undefined;
return row?.name === table;
}
function hasProviderConnection(db: SqliteDatabase, connectionId: string): boolean {
const row = db
.prepare("SELECT 1 AS ok FROM provider_connections WHERE id = ? LIMIT 1")
.get(connectionId) as { ok?: number } | undefined;
return row?.ok === 1;
}
function isValidIsoTimestamp(value: unknown): boolean {
if (typeof value !== "string" || value.trim().length === 0) return false;
return !Number.isNaN(Date.parse(value));
}
function buildRepairNote(message: string, checkedAt: string): string {
return `[db-health:${checkedAt}] ${message}`;
}
function buildDisabledCombo(row: ComboRow, checkedAt: string): JsonRecord {
const now = checkedAt;
return {
id: row.id,
name: row.name,
version: 2,
strategy: "priority",
models: [],
config: {},
isActive: false,
isHidden: false,
sortOrder: typeof row.sort_order === "number" ? row.sort_order : 0,
createdAt: row.created_at || now,
updatedAt: now,
repairNote: buildRepairNote("Combo payload was rebuilt after invalid JSON was detected.", now),
};
}
function normalizeComboModels(models: unknown): unknown[] {
return Array.isArray(models) ? models : [];
}
function repairComboRows(
db: SqliteDatabase,
rows: ComboRow[],
checkedAt: string,
options: { autoRepair: boolean }
): ComboRepairResult {
if (rows.length === 0) return { issueCount: 0, repairedCount: 0 };
const existingComboNames = new Set(rows.map((row) => row.name));
let issueCount = 0;
let repairedCount = 0;
const updateComboStmt = db.prepare("UPDATE combos SET data = ?, updated_at = ? WHERE id = ?");
for (const row of rows) {
const parsed = parseJsonRecord(row.data);
if (!parsed) {
issueCount += 1;
if (options.autoRepair) {
const repaired = buildDisabledCombo(row, checkedAt);
updateComboStmt.run(JSON.stringify(repaired), checkedAt, row.id);
repairedCount += 1;
}
continue;
}
const currentModels = normalizeComboModels(parsed.models);
if (currentModels.length === 0) continue;
const nextModels: unknown[] = [];
let removedSteps = 0;
let clearedConnectionPins = 0;
let normalizedLegacyComboRefs = 0;
for (const [index, rawStep] of currentModels.entries()) {
if (!isRecord(rawStep)) {
if (typeof rawStep === "string") {
const normalizedStep = normalizeComboStep(rawStep, {
comboName: row.name,
index,
allCombos: existingComboNames,
});
if (normalizedStep?.kind === "combo-ref") {
if (
normalizedStep.comboName === row.name ||
!existingComboNames.has(normalizedStep.comboName)
) {
removedSteps += 1;
continue;
}
nextModels.push(normalizedStep);
normalizedLegacyComboRefs += 1;
continue;
}
}
nextModels.push(rawStep);
continue;
}
if (rawStep.kind === "combo-ref") {
const comboName = toTrimmedString(rawStep.comboName);
if (!comboName || comboName === row.name || !existingComboNames.has(comboName)) {
removedSteps += 1;
continue;
}
nextModels.push(rawStep);
continue;
}
const connectionId = toTrimmedString(rawStep.connectionId);
if (connectionId && !hasProviderConnection(db, connectionId)) {
const repairedStep = { ...rawStep };
delete repairedStep.connectionId;
nextModels.push(repairedStep);
clearedConnectionPins += 1;
continue;
}
nextModels.push(rawStep);
}
if (removedSteps === 0 && clearedConnectionPins === 0 && normalizedLegacyComboRefs === 0) {
continue;
}
issueCount += removedSteps + clearedConnectionPins + normalizedLegacyComboRefs;
if (!options.autoRepair) continue;
const nextCombo = {
...parsed,
models: nextModels,
updatedAt: checkedAt,
repairNote: buildRepairNote(
[
removedSteps > 0 ? `${removedSteps} broken combo step(s) removed.` : null,
clearedConnectionPins > 0
? `${clearedConnectionPins} missing connection pin(s) cleared.`
: null,
normalizedLegacyComboRefs > 0
? `${normalizedLegacyComboRefs} legacy combo ref step(s) canonicalized.`
: null,
]
.filter(Boolean)
.join(" "),
checkedAt
),
...(nextModels.length === 0 ? { isActive: false } : {}),
};
updateComboStmt.run(JSON.stringify(nextCombo), checkedAt, row.id);
repairedCount += removedSteps + clearedConnectionPins + normalizedLegacyComboRefs;
}
return { issueCount, repairedCount };
}
function getBrokenQuotaSnapshotRowIds(db: SqliteDatabase): number[] {
if (!hasRows(db, "quota_snapshots")) return [];
const brokenRowIds = new Set<number>();
const rows = db
.prepare("SELECT id, provider, connection_id, created_at FROM quota_snapshots")
.all() as QuotaSnapshotRow[];
for (const row of rows) {
const connectionId = toTrimmedString(row.connection_id);
const missingConnection = !!connectionId && !hasProviderConnection(db, connectionId);
const invalidTimestamp = !isValidIsoTimestamp(row.created_at);
if ((missingConnection || invalidTimestamp) && typeof row.id === "number") {
brokenRowIds.add(row.id);
}
}
return Array.from(brokenRowIds);
}
function countOrphanQuotaSnapshots(db: SqliteDatabase): number {
return getBrokenQuotaSnapshotRowIds(db).length;
}
function repairQuotaSnapshots(db: SqliteDatabase): number {
if (!hasRows(db, "quota_snapshots")) return 0;
const brokenRowIds = getBrokenQuotaSnapshotRowIds(db);
if (brokenRowIds.length === 0) return 0;
const deleteByRowId = db.prepare("DELETE FROM quota_snapshots WHERE id = ?");
let repaired = 0;
for (const rowId of brokenRowIds) {
repaired += deleteByRowId.run(rowId).changes;
}
return repaired;
}
function countOrphanDomainRows(
db: SqliteDatabase,
table: "domain_budgets" | "domain_cost_history"
) {
if (!hasRows(db, table)) return 0;
const row = db
.prepare(
`SELECT COUNT(*) AS count
FROM ${table}
WHERE api_key_id NOT IN (SELECT id FROM api_keys)`
)
.get() as { count?: number } | undefined;
return row?.count || 0;
}
function repairOrphanDomainRows(
db: SqliteDatabase,
table: "domain_budgets" | "domain_cost_history"
): number {
if (!hasRows(db, table)) return 0;
return db.prepare(`DELETE FROM ${table} WHERE api_key_id NOT IN (SELECT id FROM api_keys)`).run()
.changes;
}
function countInvalidJsonRows(
db: SqliteDatabase,
table: "domain_fallback_chains" | "domain_lockout_state" | "domain_circuit_breakers",
column: "chain" | "attempts" | "options"
): number {
if (!hasRows(db, table)) return 0;
const rows = db.prepare(`SELECT ${column} FROM ${table}`).all() as Array<Record<string, unknown>>;
let invalid = 0;
for (const row of rows) {
const raw = row[column];
if (raw == null && column === "options") continue;
if (typeof raw !== "string") {
invalid += 1;
continue;
}
try {
JSON.parse(raw);
} catch {
invalid += 1;
}
}
return invalid;
}
function repairInvalidJsonRows(
db: SqliteDatabase,
table: "domain_fallback_chains" | "domain_lockout_state" | "domain_circuit_breakers",
column: "chain" | "attempts" | "options"
): number {
if (!hasRows(db, table)) return 0;
const rows = db.prepare(`SELECT rowid, ${column} FROM ${table}`).all() as Array<{
rowid: number;
[key: string]: unknown;
}>;
const deleteByRowId = db.prepare(`DELETE FROM ${table} WHERE rowid = ?`);
const clearOptionsByRowId = db.prepare(
"UPDATE domain_circuit_breakers SET options = NULL WHERE rowid = ?"
);
let repaired = 0;
for (const row of rows) {
const raw = row[column];
if (raw == null && table === "domain_circuit_breakers") {
continue;
}
if (typeof raw !== "string") {
if (table === "domain_circuit_breakers") {
repaired += clearOptionsByRowId.run(row.rowid).changes;
continue;
}
deleteByRowId.run(row.rowid);
repaired += 1;
continue;
}
try {
JSON.parse(raw);
} catch {
if (table === "domain_circuit_breakers") {
repaired += clearOptionsByRowId.run(row.rowid).changes;
continue;
}
deleteByRowId.run(row.rowid);
repaired += 1;
}
}
return repaired;
}
function getSchemaVersionIssueCount(db: SqliteDatabase, expectedSchemaVersion: string): number {
if (!hasRows(db, "db_meta")) return 0;
const row = db.prepare("SELECT value FROM db_meta WHERE key = 'schema_version'").get() as
| { value?: string | null }
| undefined;
const current = typeof row?.value === "string" ? row.value : null;
return current === expectedSchemaVersion ? 0 : 1;
}
function repairSchemaVersion(db: SqliteDatabase, expectedSchemaVersion: string): number {
if (!hasRows(db, "db_meta")) return 0;
return db
.prepare("INSERT OR REPLACE INTO db_meta (key, value) VALUES ('schema_version', ?)")
.run(expectedSchemaVersion).changes;
}
export function runDbHealthCheck(
db: SqliteDatabase,
options: RunDbHealthCheckOptions = {}
): DbHealthCheckResult {
const autoRepair = options.autoRepair === true;
const expectedSchemaVersion = options.expectedSchemaVersion || "1";
const checkedAt = new Date().toISOString();
const issues: DbHealthIssue[] = [];
let repairedCount = 0;
let backupCreated = false;
let backupAttempted = false;
const ensureBackupBeforeRepair = () => {
if (!autoRepair || backupAttempted || typeof options.createBackupBeforeRepair !== "function") {
return;
}
backupAttempted = true;
backupCreated = options.createBackupBeforeRepair();
};
// Use quick_check instead of integrity_check on startup — integrity_check
// does a full page-by-page scan that can take minutes on a fragmented WAL,
// causing 7+ minute boot times. quick_check still catches corruption but
// skips deep index verification, reducing I/O to seconds.
// Skip entirely when skipIntegrityCheck is set (env OMNIROUTE_SKIP_DB_HEALTHCHECK=1).
if (!options.skipIntegrityCheck) {
const integrityCheck = db.pragma("quick_check") as Array<{ quick_check?: string }>;
if (integrityCheck[0]?.quick_check !== "ok") {
issues.push({
type: "integrity_check_failed",
table: "sqlite",
description: "SQLite integrity_check returned a non-ok status.",
count: 1,
});
}
}
if (hasRows(db, "combos")) {
const comboRows = db
.prepare(
"SELECT id, name, data, sort_order, created_at, updated_at FROM combos ORDER BY name COLLATE NOCASE ASC"
)
.all() as ComboRow[];
const comboRepair = repairComboRows(db, comboRows, checkedAt, { autoRepair });
if (comboRepair.issueCount > 0) {
issues.push({
type: "broken_reference",
table: "combos",
description:
"Combos contained broken combo references, legacy combo refs, invalid JSON, or pinned connections that no longer exist.",
count: comboRepair.issueCount,
});
if (autoRepair) {
ensureBackupBeforeRepair();
repairedCount += comboRepair.repairedCount;
}
}
}
const orphanQuotaCount = countOrphanQuotaSnapshots(db);
if (orphanQuotaCount > 0) {
issues.push({
type: "stale_snapshot",
table: "quota_snapshots",
description:
"Quota snapshots referenced missing connections or contained invalid timestamps.",
count: orphanQuotaCount,
});
if (autoRepair) {
ensureBackupBeforeRepair();
repairedCount += repairQuotaSnapshots(db);
}
}
const orphanBudgets = countOrphanDomainRows(db, "domain_budgets");
if (orphanBudgets > 0) {
issues.push({
type: "broken_reference",
table: "domain_budgets",
description: "Domain budgets referenced API keys that no longer exist.",
count: orphanBudgets,
});
if (autoRepair) {
ensureBackupBeforeRepair();
repairedCount += repairOrphanDomainRows(db, "domain_budgets");
}
}
const orphanCostHistory = countOrphanDomainRows(db, "domain_cost_history");
if (orphanCostHistory > 0) {
issues.push({
type: "broken_reference",
table: "domain_cost_history",
description: "Domain cost history referenced API keys that no longer exist.",
count: orphanCostHistory,
});
if (autoRepair) {
ensureBackupBeforeRepair();
repairedCount += repairOrphanDomainRows(db, "domain_cost_history");
}
}
const invalidFallbackChains = countInvalidJsonRows(db, "domain_fallback_chains", "chain");
if (invalidFallbackChains > 0) {
issues.push({
type: "invalid_state",
table: "domain_fallback_chains",
description: "Fallback chain rows contained invalid JSON payloads.",
count: invalidFallbackChains,
});
if (autoRepair) {
ensureBackupBeforeRepair();
repairedCount += repairInvalidJsonRows(db, "domain_fallback_chains", "chain");
}
}
const invalidLockoutState = countInvalidJsonRows(db, "domain_lockout_state", "attempts");
if (invalidLockoutState > 0) {
issues.push({
type: "invalid_state",
table: "domain_lockout_state",
description: "Lockout state rows contained invalid JSON payloads.",
count: invalidLockoutState,
});
if (autoRepair) {
ensureBackupBeforeRepair();
repairedCount += repairInvalidJsonRows(db, "domain_lockout_state", "attempts");
}
}
const invalidBreakerOptions = countInvalidJsonRows(db, "domain_circuit_breakers", "options");
if (invalidBreakerOptions > 0) {
issues.push({
type: "invalid_state",
table: "domain_circuit_breakers",
description: "Circuit breaker option payloads were invalid JSON.",
count: invalidBreakerOptions,
});
if (autoRepair) {
ensureBackupBeforeRepair();
repairedCount += repairInvalidJsonRows(db, "domain_circuit_breakers", "options");
}
}
const schemaVersionIssues = getSchemaVersionIssueCount(db, expectedSchemaVersion);
if (schemaVersionIssues > 0) {
issues.push({
type: "invalid_state",
table: "db_meta",
description: `db_meta.schema_version did not match expected version ${expectedSchemaVersion}.`,
count: schemaVersionIssues,
});
if (autoRepair) {
ensureBackupBeforeRepair();
repairedCount += repairSchemaVersion(db, expectedSchemaVersion);
}
}
return {
isHealthy: issues.length === 0,
issues,
repairedCount,
backupCreated,
autoRepair,
checkedAt,
};
}