Spaces:
Paused
Paused
| /** | |
| * βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| * β RESILIENCE UTILITIES β | |
| * βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| * β Circuit Breaker, Retry Logic, Timeout handling β | |
| * β Beskytter systemet mod cascading failures β | |
| * βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| */ | |
| import { logger } from './logger.js'; | |
| const log = logger.child({ module: 'Resilience' }); | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // CIRCUIT BREAKER | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| export type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN'; | |
| export interface CircuitBreakerOptions { | |
| name: string; | |
| failureThreshold?: number; // Antal fejl fΓΈr circuit Γ₯bner (default: 5) | |
| successThreshold?: number; // Antal successer i HALF_OPEN fΓΈr circuit lukker (default: 2) | |
| timeout?: number; // Tid i ms fΓΈr OPEN -> HALF_OPEN (default: 30000) | |
| resetTimeout?: number; // Tid fΓΈr failure count nulstilles (default: 60000) | |
| } | |
| export class CircuitBreaker { | |
| private state: CircuitState = 'CLOSED'; | |
| private failureCount = 0; | |
| private successCount = 0; | |
| private lastFailureTime = 0; | |
| private nextAttemptTime = 0; | |
| private readonly name: string; | |
| private readonly failureThreshold: number; | |
| private readonly successThreshold: number; | |
| private readonly timeout: number; | |
| private readonly resetTimeout: number; | |
| constructor(options: CircuitBreakerOptions) { | |
| this.name = options.name; | |
| this.failureThreshold = options.failureThreshold ?? 5; | |
| this.successThreshold = options.successThreshold ?? 2; | |
| this.timeout = options.timeout ?? 30000; | |
| this.resetTimeout = options.resetTimeout ?? 60000; | |
| } | |
| /** | |
| * Execute a function through the circuit breaker | |
| */ | |
| async execute<T>(fn: () => Promise<T>): Promise<T> { | |
| // Check if circuit should transition from OPEN to HALF_OPEN | |
| if (this.state === 'OPEN') { | |
| if (Date.now() >= this.nextAttemptTime) { | |
| this.state = 'HALF_OPEN'; | |
| log.info(`Circuit [${this.name}] transitioning to HALF_OPEN`); | |
| } else { | |
| const waitTime = Math.ceil((this.nextAttemptTime - Date.now()) / 1000); | |
| throw new CircuitOpenError( | |
| `Circuit [${this.name}] is OPEN. Retry in ${waitTime}s` | |
| ); | |
| } | |
| } | |
| try { | |
| const result = await fn(); | |
| this.onSuccess(); | |
| return result; | |
| } catch (error) { | |
| this.onFailure(); | |
| throw error; | |
| } | |
| } | |
| private onSuccess(): void { | |
| if (this.state === 'HALF_OPEN') { | |
| this.successCount++; | |
| if (this.successCount >= this.successThreshold) { | |
| this.state = 'CLOSED'; | |
| this.failureCount = 0; | |
| this.successCount = 0; | |
| log.info(`Circuit [${this.name}] CLOSED - service recovered`); | |
| } | |
| } else if (this.state === 'CLOSED') { | |
| // Reset failure count after successful call if enough time passed | |
| if (Date.now() - this.lastFailureTime > this.resetTimeout) { | |
| this.failureCount = 0; | |
| } | |
| } | |
| } | |
| private onFailure(): void { | |
| this.failureCount++; | |
| this.lastFailureTime = Date.now(); | |
| this.successCount = 0; | |
| if (this.state === 'HALF_OPEN') { | |
| // Immediately open on failure in HALF_OPEN | |
| this.state = 'OPEN'; | |
| this.nextAttemptTime = Date.now() + this.timeout; | |
| log.warn(`Circuit [${this.name}] OPEN - failed during recovery test`); | |
| } else if (this.failureCount >= this.failureThreshold) { | |
| this.state = 'OPEN'; | |
| this.nextAttemptTime = Date.now() + this.timeout; | |
| log.warn(`Circuit [${this.name}] OPEN - threshold reached (${this.failureCount} failures)`); | |
| } | |
| } | |
| getState(): CircuitState { | |
| return this.state; | |
| } | |
| getStats() { | |
| return { | |
| name: this.name, | |
| state: this.state, | |
| failureCount: this.failureCount, | |
| successCount: this.successCount, | |
| lastFailureTime: this.lastFailureTime, | |
| nextAttemptTime: this.nextAttemptTime | |
| }; | |
| } | |
| /** | |
| * Manually reset the circuit breaker | |
| */ | |
| reset(): void { | |
| this.state = 'CLOSED'; | |
| this.failureCount = 0; | |
| this.successCount = 0; | |
| log.info(`Circuit [${this.name}] manually reset`); | |
| } | |
| } | |
| export class CircuitOpenError extends Error { | |
| constructor(message: string) { | |
| super(message); | |
| this.name = 'CircuitOpenError'; | |
| } | |
| } | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // RETRY WITH EXPONENTIAL BACKOFF | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| export interface RetryOptions { | |
| maxAttempts?: number; // Max forsΓΈg (default: 3) | |
| initialDelay?: number; // Start delay i ms (default: 1000) | |
| maxDelay?: number; // Max delay i ms (default: 30000) | |
| backoffMultiplier?: number; // Multiplier for delay (default: 2) | |
| jitter?: boolean; // TilfΓΈj random jitter (default: true) | |
| retryCondition?: (error: any) => boolean; // HvornΓ₯r skal vi retry | |
| } | |
| export async function withRetry<T>( | |
| fn: () => Promise<T>, | |
| options: RetryOptions = {} | |
| ): Promise<T> { | |
| const { | |
| maxAttempts = 3, | |
| initialDelay = 1000, | |
| maxDelay = 30000, | |
| backoffMultiplier = 2, | |
| jitter = true, | |
| retryCondition = () => true | |
| } = options; | |
| let lastError: any; | |
| let delay = initialDelay; | |
| for (let attempt = 1; attempt <= maxAttempts; attempt++) { | |
| try { | |
| return await fn(); | |
| } catch (error) { | |
| lastError = error; | |
| // Don't retry circuit open errors | |
| if (error instanceof CircuitOpenError) { | |
| throw error; | |
| } | |
| // Check if we should retry | |
| if (!retryCondition(error)) { | |
| throw error; | |
| } | |
| // Last attempt - throw | |
| if (attempt === maxAttempts) { | |
| log.error(`All ${maxAttempts} retry attempts failed`, { error }); | |
| throw error; | |
| } | |
| // Calculate delay with optional jitter | |
| let waitTime = Math.min(delay, maxDelay); | |
| if (jitter) { | |
| waitTime = waitTime * (0.5 + Math.random()); | |
| } | |
| log.warn(`Attempt ${attempt}/${maxAttempts} failed, retrying in ${Math.round(waitTime)}ms`, { | |
| error: error instanceof Error ? error.message : String(error) | |
| }); | |
| await sleep(waitTime); | |
| delay *= backoffMultiplier; | |
| } | |
| } | |
| throw lastError; | |
| } | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // TIMEOUT WRAPPER | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| export class TimeoutError extends Error { | |
| constructor(message: string) { | |
| super(message); | |
| this.name = 'TimeoutError'; | |
| } | |
| } | |
| export async function withTimeout<T>( | |
| fn: () => Promise<T>, | |
| timeoutMs: number, | |
| operation = 'Operation' | |
| ): Promise<T> { | |
| return Promise.race([ | |
| fn(), | |
| new Promise<never>((_, reject) => | |
| setTimeout( | |
| () => reject(new TimeoutError(`${operation} timed out after ${timeoutMs}ms`)), | |
| timeoutMs | |
| ) | |
| ) | |
| ]); | |
| } | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // COMBINED: RESILIENT CALL | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| export interface ResilientCallOptions extends RetryOptions { | |
| timeout?: number; | |
| circuitBreaker?: CircuitBreaker; | |
| } | |
| /** | |
| * Execute a function with circuit breaker, retry, and timeout protection | |
| */ | |
| export async function resilientCall<T>( | |
| fn: () => Promise<T>, | |
| options: ResilientCallOptions = {} | |
| ): Promise<T> { | |
| const { timeout, circuitBreaker, ...retryOptions } = options; | |
| // Wrap with timeout if specified | |
| let wrappedFn = fn; | |
| if (timeout) { | |
| wrappedFn = () => withTimeout(fn, timeout); | |
| } | |
| // Wrap with circuit breaker if specified | |
| if (circuitBreaker) { | |
| const originalFn = wrappedFn; | |
| wrappedFn = () => circuitBreaker.execute(originalFn); | |
| } | |
| // Wrap with retry | |
| return withRetry(wrappedFn, retryOptions); | |
| } | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // CIRCUIT BREAKER REGISTRY | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| const circuitBreakers = new Map<string, CircuitBreaker>(); | |
| /** | |
| * Get or create a circuit breaker by name | |
| */ | |
| export function getCircuitBreaker( | |
| name: string, | |
| options?: Omit<CircuitBreakerOptions, 'name'> | |
| ): CircuitBreaker { | |
| let cb = circuitBreakers.get(name); | |
| if (!cb) { | |
| cb = new CircuitBreaker({ name, ...options }); | |
| circuitBreakers.set(name, cb); | |
| } | |
| return cb; | |
| } | |
| /** | |
| * Get status of all circuit breakers | |
| */ | |
| export function getAllCircuitBreakerStats() { | |
| return Array.from(circuitBreakers.values()).map(cb => cb.getStats()); | |
| } | |
| /** | |
| * Reset all circuit breakers | |
| */ | |
| export function resetAllCircuitBreakers(): void { | |
| circuitBreakers.forEach(cb => cb.reset()); | |
| } | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // UTILITY | |
| // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| export function sleep(ms: number): Promise<void> { | |
| return new Promise(resolve => setTimeout(resolve, ms)); | |
| } | |
| /** | |
| * Check if an error is retryable (network errors, timeouts, 5xx) | |
| */ | |
| export function isRetryableError(error: any): boolean { | |
| // Network errors | |
| if (error.code === 'ECONNREFUSED' || | |
| error.code === 'ENOTFOUND' || | |
| error.code === 'ETIMEDOUT' || | |
| error.code === 'ECONNRESET') { | |
| return true; | |
| } | |
| // Timeout errors | |
| if (error instanceof TimeoutError) { | |
| return true; | |
| } | |
| // HTTP 5xx errors | |
| if (error.response?.status >= 500) { | |
| return true; | |
| } | |
| // HTTP 429 (rate limited) | |
| if (error.response?.status === 429) { | |
| return true; | |
| } | |
| return false; | |
| } | |