/** * ╔═══════════════════════════════════════════════════════════════════════════╗ * ║ RESILIENCE UTILITIES ║ * ║═══════════════════════════════════════════════════════════════════════════║ * ║ Circuit Breaker, Retry Logic, Timeout handling ║ * ║ Beskytter systemet mod cascading failures ║ * ╚═══════════════════════════════════════════════════════════════════════════╝ */ import { logger } from './logger.js'; const log = logger.child({ module: 'Resilience' }); // ═══════════════════════════════════════════════════════════════════════════ // CIRCUIT BREAKER // ═══════════════════════════════════════════════════════════════════════════ export type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN'; export interface CircuitBreakerOptions { name: string; failureThreshold?: number; // Antal fejl før circuit åbner (default: 5) successThreshold?: number; // Antal successer i HALF_OPEN før circuit lukker (default: 2) timeout?: number; // Tid i ms før OPEN -> HALF_OPEN (default: 30000) resetTimeout?: number; // Tid før failure count nulstilles (default: 60000) } export class CircuitBreaker { private state: CircuitState = 'CLOSED'; private failureCount = 0; private successCount = 0; private lastFailureTime = 0; private nextAttemptTime = 0; private readonly name: string; private readonly failureThreshold: number; private readonly successThreshold: number; private readonly timeout: number; private readonly resetTimeout: number; constructor(options: CircuitBreakerOptions) { this.name = options.name; this.failureThreshold = options.failureThreshold ?? 5; this.successThreshold = options.successThreshold ?? 2; this.timeout = options.timeout ?? 30000; this.resetTimeout = options.resetTimeout ?? 60000; } /** * Execute a function through the circuit breaker */ async execute(fn: () => Promise): Promise { // Check if circuit should transition from OPEN to HALF_OPEN if (this.state === 'OPEN') { if (Date.now() >= this.nextAttemptTime) { this.state = 'HALF_OPEN'; log.info(`Circuit [${this.name}] transitioning to HALF_OPEN`); } else { const waitTime = Math.ceil((this.nextAttemptTime - Date.now()) / 1000); throw new CircuitOpenError( `Circuit [${this.name}] is OPEN. Retry in ${waitTime}s` ); } } try { const result = await fn(); this.onSuccess(); return result; } catch (error) { this.onFailure(); throw error; } } private onSuccess(): void { if (this.state === 'HALF_OPEN') { this.successCount++; if (this.successCount >= this.successThreshold) { this.state = 'CLOSED'; this.failureCount = 0; this.successCount = 0; log.info(`Circuit [${this.name}] CLOSED - service recovered`); } } else if (this.state === 'CLOSED') { // Reset failure count after successful call if enough time passed if (Date.now() - this.lastFailureTime > this.resetTimeout) { this.failureCount = 0; } } } private onFailure(): void { this.failureCount++; this.lastFailureTime = Date.now(); this.successCount = 0; if (this.state === 'HALF_OPEN') { // Immediately open on failure in HALF_OPEN this.state = 'OPEN'; this.nextAttemptTime = Date.now() + this.timeout; log.warn(`Circuit [${this.name}] OPEN - failed during recovery test`); } else if (this.failureCount >= this.failureThreshold) { this.state = 'OPEN'; this.nextAttemptTime = Date.now() + this.timeout; log.warn(`Circuit [${this.name}] OPEN - threshold reached (${this.failureCount} failures)`); } } getState(): CircuitState { return this.state; } getStats() { return { name: this.name, state: this.state, failureCount: this.failureCount, successCount: this.successCount, lastFailureTime: this.lastFailureTime, nextAttemptTime: this.nextAttemptTime }; } /** * Manually reset the circuit breaker */ reset(): void { this.state = 'CLOSED'; this.failureCount = 0; this.successCount = 0; log.info(`Circuit [${this.name}] manually reset`); } } export class CircuitOpenError extends Error { constructor(message: string) { super(message); this.name = 'CircuitOpenError'; } } // ═══════════════════════════════════════════════════════════════════════════ // RETRY WITH EXPONENTIAL BACKOFF // ═══════════════════════════════════════════════════════════════════════════ export interface RetryOptions { maxAttempts?: number; // Max forsøg (default: 3) initialDelay?: number; // Start delay i ms (default: 1000) maxDelay?: number; // Max delay i ms (default: 30000) backoffMultiplier?: number; // Multiplier for delay (default: 2) jitter?: boolean; // Tilføj random jitter (default: true) retryCondition?: (error: any) => boolean; // Hvornår skal vi retry } export async function withRetry( fn: () => Promise, options: RetryOptions = {} ): Promise { const { maxAttempts = 3, initialDelay = 1000, maxDelay = 30000, backoffMultiplier = 2, jitter = true, retryCondition = () => true } = options; let lastError: any; let delay = initialDelay; for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { return await fn(); } catch (error) { lastError = error; // Don't retry circuit open errors if (error instanceof CircuitOpenError) { throw error; } // Check if we should retry if (!retryCondition(error)) { throw error; } // Last attempt - throw if (attempt === maxAttempts) { log.error(`All ${maxAttempts} retry attempts failed`, { error }); throw error; } // Calculate delay with optional jitter let waitTime = Math.min(delay, maxDelay); if (jitter) { waitTime = waitTime * (0.5 + Math.random()); } log.warn(`Attempt ${attempt}/${maxAttempts} failed, retrying in ${Math.round(waitTime)}ms`, { error: error instanceof Error ? error.message : String(error) }); await sleep(waitTime); delay *= backoffMultiplier; } } throw lastError; } // ═══════════════════════════════════════════════════════════════════════════ // TIMEOUT WRAPPER // ═══════════════════════════════════════════════════════════════════════════ export class TimeoutError extends Error { constructor(message: string) { super(message); this.name = 'TimeoutError'; } } export async function withTimeout( fn: () => Promise, timeoutMs: number, operation = 'Operation' ): Promise { return Promise.race([ fn(), new Promise((_, reject) => setTimeout( () => reject(new TimeoutError(`${operation} timed out after ${timeoutMs}ms`)), timeoutMs ) ) ]); } // ═══════════════════════════════════════════════════════════════════════════ // COMBINED: RESILIENT CALL // ═══════════════════════════════════════════════════════════════════════════ export interface ResilientCallOptions extends RetryOptions { timeout?: number; circuitBreaker?: CircuitBreaker; } /** * Execute a function with circuit breaker, retry, and timeout protection */ export async function resilientCall( fn: () => Promise, options: ResilientCallOptions = {} ): Promise { const { timeout, circuitBreaker, ...retryOptions } = options; // Wrap with timeout if specified let wrappedFn = fn; if (timeout) { wrappedFn = () => withTimeout(fn, timeout); } // Wrap with circuit breaker if specified if (circuitBreaker) { const originalFn = wrappedFn; wrappedFn = () => circuitBreaker.execute(originalFn); } // Wrap with retry return withRetry(wrappedFn, retryOptions); } // ═══════════════════════════════════════════════════════════════════════════ // CIRCUIT BREAKER REGISTRY // ═══════════════════════════════════════════════════════════════════════════ const circuitBreakers = new Map(); /** * Get or create a circuit breaker by name */ export function getCircuitBreaker( name: string, options?: Omit ): CircuitBreaker { let cb = circuitBreakers.get(name); if (!cb) { cb = new CircuitBreaker({ name, ...options }); circuitBreakers.set(name, cb); } return cb; } /** * Get status of all circuit breakers */ export function getAllCircuitBreakerStats() { return Array.from(circuitBreakers.values()).map(cb => cb.getStats()); } /** * Reset all circuit breakers */ export function resetAllCircuitBreakers(): void { circuitBreakers.forEach(cb => cb.reset()); } // ═══════════════════════════════════════════════════════════════════════════ // UTILITY // ═══════════════════════════════════════════════════════════════════════════ export function sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } /** * Check if an error is retryable (network errors, timeouts, 5xx) */ export function isRetryableError(error: any): boolean { // Network errors if (error.code === 'ECONNREFUSED' || error.code === 'ENOTFOUND' || error.code === 'ETIMEDOUT' || error.code === 'ECONNRESET') { return true; } // Timeout errors if (error instanceof TimeoutError) { return true; } // HTTP 5xx errors if (error.response?.status >= 500) { return true; } // HTTP 429 (rate limited) if (error.response?.status === 429) { return true; } return false; }