| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| import { getProviderNodes } from "@/lib/localDb"; |
|
|
| |
|
|
| export interface HealthStatus { |
| nodeId: string; |
| prefix: string; |
| isHealthy: boolean; |
| lastCheck: Date; |
| lastError?: string; |
| consecutiveFailures: number; |
| responseTimeMs?: number; |
| } |
|
|
| |
|
|
| const BACKOFF_SCHEDULE = [30_000, 60_000, 120_000, 300_000]; |
| const CHECK_TIMEOUT_MS = 5_000; |
| const INITIAL_DELAY_MS = 15_000; |
| const LOG_PREFIX = "[LocalHealthCheck]"; |
| const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]); |
|
|
| |
|
|
| declare global { |
| var __omnirouteLocalHC: |
| | { |
| initialized: boolean; |
| sweepTimer: ReturnType<typeof setTimeout> | null; |
| healthCache: Map<string, HealthStatus>; |
| sweepInProgress: boolean; |
| } |
| | undefined; |
| } |
|
|
| function getLHCState() { |
| if (!globalThis.__omnirouteLocalHC) { |
| globalThis.__omnirouteLocalHC = { |
| initialized: false, |
| sweepTimer: null, |
| healthCache: new Map(), |
| sweepInProgress: false, |
| }; |
| } |
| return globalThis.__omnirouteLocalHC; |
| } |
|
|
| const healthCache = getLHCState().healthCache; |
|
|
| |
|
|
| function isEnvFlagEnabled(name: string): boolean { |
| const value = process.env[name]; |
| if (!value) return false; |
| return TRUE_ENV_VALUES.has(value.trim().toLowerCase()); |
| } |
|
|
| function isLocalHealthCheckDisabled(): boolean { |
| return isEnvFlagEnabled("OMNIROUTE_DISABLE_LOCAL_HEALTHCHECK") || process.env.NODE_ENV === "test"; |
| } |
|
|
| function isLocalhostUrl(baseUrl: string): boolean { |
| try { |
| const u = new URL(baseUrl); |
| |
| if (u.username || u.password) return false; |
| |
| |
| |
| return ( |
| u.hostname === "localhost" || |
| u.hostname === "127.0.0.1" || |
| /^172\.(1[6-9]|2[0-9]|3[0-1])\.\d{1,3}\.\d{1,3}$/.test(u.hostname) |
| ); |
| } catch { |
| return false; |
| } |
| } |
|
|
| function getNextInterval(failures: number): number { |
| return BACKOFF_SCHEDULE[Math.min(failures, BACKOFF_SCHEDULE.length - 1)]; |
| } |
|
|
| |
|
|
| async function checkNode(node: { |
| id: string; |
| prefix: string; |
| baseUrl: string; |
| }): Promise<HealthStatus> { |
| const url = `${node.baseUrl.replace(/\/+$/, "")}/models`; |
| const start = Date.now(); |
| const prev = healthCache.get(node.id); |
|
|
| try { |
| const res = await fetch(url, { signal: AbortSignal.timeout(CHECK_TIMEOUT_MS) }); |
| |
| res.body?.cancel().catch(() => {}); |
| const isHealthy = res.ok || res.status === 401; |
| return { |
| nodeId: node.id, |
| prefix: node.prefix, |
| isHealthy, |
| lastCheck: new Date(), |
| consecutiveFailures: isHealthy ? 0 : (prev?.consecutiveFailures ?? 0) + 1, |
| responseTimeMs: Date.now() - start, |
| lastError: isHealthy ? undefined : `HTTP ${res.status}`, |
| }; |
| } catch (err: unknown) { |
| const message = err instanceof Error ? err.message : "Connection failed"; |
| return { |
| nodeId: node.id, |
| prefix: node.prefix, |
| isHealthy: false, |
| lastCheck: new Date(), |
| consecutiveFailures: (prev?.consecutiveFailures ?? 0) + 1, |
| responseTimeMs: Date.now() - start, |
| lastError: message, |
| }; |
| } |
| } |
|
|
| |
| export async function sweep(): Promise<void> { |
| const state = getLHCState(); |
| if (state.sweepInProgress) return; |
| state.sweepInProgress = true; |
|
|
| try { |
| let nodes: Array<{ id: string; prefix: string; baseUrl: string }>; |
| try { |
| const raw = await getProviderNodes(); |
| nodes = (Array.isArray(raw) ? raw : []).filter( |
| (n: Record<string, unknown>) => |
| typeof n.baseUrl === "string" && isLocalhostUrl(n.baseUrl as string) |
| ) as Array<{ id: string; prefix: string; baseUrl: string }>; |
| } catch (err) { |
| console.error(LOG_PREFIX, "Failed to load provider_nodes:", err); |
| return; |
| } |
|
|
| |
| const currentNodeIds = new Set(nodes.map((n) => n.id)); |
| for (const key of healthCache.keys()) { |
| if (!currentNodeIds.has(key)) healthCache.delete(key); |
| } |
|
|
| if (nodes.length === 0) return; |
|
|
| const results = await Promise.allSettled(nodes.map((node) => checkNode(node))); |
|
|
| for (const result of results) { |
| if (result.status === "fulfilled") { |
| const status = result.value; |
| const prev = healthCache.get(status.nodeId); |
|
|
| |
| if (prev && prev.isHealthy !== status.isHealthy) { |
| const emoji = status.isHealthy ? "β
" : "β"; |
| console.log( |
| LOG_PREFIX, |
| `${emoji} ${status.prefix} is now ${status.isHealthy ? "healthy" : "unhealthy"}${status.lastError ? ` (${status.lastError})` : ""} [${status.responseTimeMs}ms]` |
| ); |
| } |
|
|
| healthCache.set(status.nodeId, status); |
| } |
| } |
| } finally { |
| state.sweepInProgress = false; |
| scheduleSweep(); |
| } |
| } |
|
|
| function scheduleSweep(): void { |
| const state = getLHCState(); |
| if (!state.initialized) return; |
| if (state.sweepTimer) clearTimeout(state.sweepTimer); |
|
|
| |
| let maxFailures = 0; |
| for (const status of healthCache.values()) { |
| if (status.consecutiveFailures > maxFailures) { |
| maxFailures = status.consecutiveFailures; |
| } |
| } |
|
|
| const interval = getNextInterval(maxFailures); |
| state.sweepTimer = setTimeout(sweep, interval); |
| } |
|
|
| |
|
|
| |
| export function getHealthStatus(nodeId: string): HealthStatus | undefined { |
| return healthCache.get(nodeId); |
| } |
|
|
| |
| export function isNodeHealthy(nodeId: string): boolean { |
| const status = healthCache.get(nodeId); |
| return status?.isHealthy ?? true; |
| } |
|
|
| |
| export function getAllHealthStatuses(): Record<string, HealthStatus> { |
| return Object.fromEntries(healthCache); |
| } |
|
|
| |
| export function initLocalHealthCheck(): void { |
| const state = getLHCState(); |
| if (state.initialized || isLocalHealthCheckDisabled()) return; |
| state.initialized = true; |
|
|
| console.log( |
| LOG_PREFIX, |
| `Starting local provider health check (initial delay ${INITIAL_DELAY_MS / 1000}s)` |
| ); |
|
|
| state.sweepTimer = setTimeout(() => { |
| sweep().catch((err) => console.error(LOG_PREFIX, "Initial sweep failed:", err)); |
| }, INITIAL_DELAY_MS); |
| } |
|
|
| |
| export function stopLocalHealthCheck(): void { |
| const state = getLHCState(); |
| if (state.sweepTimer) { |
| clearTimeout(state.sweepTimer); |
| state.sweepTimer = null; |
| } |
| state.initialized = false; |
| } |
|
|
| |
| initLocalHealthCheck(); |
|
|