Kraft102's picture
Initial deployment - WidgeTDC Cortex Backend v2.1.0
529090e
/**
* ╔═══════════════════════════════════════════════════════════════════════════╗
* β•‘ RESILIENCE UTILITIES β•‘
* ║═══════════════════════════════════════════════════════════════════════════║
* β•‘ Circuit Breaker, Retry Logic, Timeout handling β•‘
* β•‘ Beskytter systemet mod cascading failures β•‘
* β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
*/
import { logger } from './logger.js';
const log = logger.child({ module: 'Resilience' });
// ═══════════════════════════════════════════════════════════════════════════
// CIRCUIT BREAKER
// ═══════════════════════════════════════════════════════════════════════════
export type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
export interface CircuitBreakerOptions {
name: string;
failureThreshold?: number; // Antal fejl fΓΈr circuit Γ₯bner (default: 5)
successThreshold?: number; // Antal successer i HALF_OPEN fΓΈr circuit lukker (default: 2)
timeout?: number; // Tid i ms fΓΈr OPEN -> HALF_OPEN (default: 30000)
resetTimeout?: number; // Tid fΓΈr failure count nulstilles (default: 60000)
}
export class CircuitBreaker {
private state: CircuitState = 'CLOSED';
private failureCount = 0;
private successCount = 0;
private lastFailureTime = 0;
private nextAttemptTime = 0;
private readonly name: string;
private readonly failureThreshold: number;
private readonly successThreshold: number;
private readonly timeout: number;
private readonly resetTimeout: number;
constructor(options: CircuitBreakerOptions) {
this.name = options.name;
this.failureThreshold = options.failureThreshold ?? 5;
this.successThreshold = options.successThreshold ?? 2;
this.timeout = options.timeout ?? 30000;
this.resetTimeout = options.resetTimeout ?? 60000;
}
/**
* Execute a function through the circuit breaker
*/
async execute<T>(fn: () => Promise<T>): Promise<T> {
// Check if circuit should transition from OPEN to HALF_OPEN
if (this.state === 'OPEN') {
if (Date.now() >= this.nextAttemptTime) {
this.state = 'HALF_OPEN';
log.info(`Circuit [${this.name}] transitioning to HALF_OPEN`);
} else {
const waitTime = Math.ceil((this.nextAttemptTime - Date.now()) / 1000);
throw new CircuitOpenError(
`Circuit [${this.name}] is OPEN. Retry in ${waitTime}s`
);
}
}
try {
const result = await fn();
this.onSuccess();
return result;
} catch (error) {
this.onFailure();
throw error;
}
}
private onSuccess(): void {
if (this.state === 'HALF_OPEN') {
this.successCount++;
if (this.successCount >= this.successThreshold) {
this.state = 'CLOSED';
this.failureCount = 0;
this.successCount = 0;
log.info(`Circuit [${this.name}] CLOSED - service recovered`);
}
} else if (this.state === 'CLOSED') {
// Reset failure count after successful call if enough time passed
if (Date.now() - this.lastFailureTime > this.resetTimeout) {
this.failureCount = 0;
}
}
}
private onFailure(): void {
this.failureCount++;
this.lastFailureTime = Date.now();
this.successCount = 0;
if (this.state === 'HALF_OPEN') {
// Immediately open on failure in HALF_OPEN
this.state = 'OPEN';
this.nextAttemptTime = Date.now() + this.timeout;
log.warn(`Circuit [${this.name}] OPEN - failed during recovery test`);
} else if (this.failureCount >= this.failureThreshold) {
this.state = 'OPEN';
this.nextAttemptTime = Date.now() + this.timeout;
log.warn(`Circuit [${this.name}] OPEN - threshold reached (${this.failureCount} failures)`);
}
}
getState(): CircuitState {
return this.state;
}
getStats() {
return {
name: this.name,
state: this.state,
failureCount: this.failureCount,
successCount: this.successCount,
lastFailureTime: this.lastFailureTime,
nextAttemptTime: this.nextAttemptTime
};
}
/**
* Manually reset the circuit breaker
*/
reset(): void {
this.state = 'CLOSED';
this.failureCount = 0;
this.successCount = 0;
log.info(`Circuit [${this.name}] manually reset`);
}
}
export class CircuitOpenError extends Error {
constructor(message: string) {
super(message);
this.name = 'CircuitOpenError';
}
}
// ═══════════════════════════════════════════════════════════════════════════
// RETRY WITH EXPONENTIAL BACKOFF
// ═══════════════════════════════════════════════════════════════════════════
export interface RetryOptions {
maxAttempts?: number; // Max forsΓΈg (default: 3)
initialDelay?: number; // Start delay i ms (default: 1000)
maxDelay?: number; // Max delay i ms (default: 30000)
backoffMultiplier?: number; // Multiplier for delay (default: 2)
jitter?: boolean; // TilfΓΈj random jitter (default: true)
retryCondition?: (error: any) => boolean; // HvornΓ₯r skal vi retry
}
export async function withRetry<T>(
fn: () => Promise<T>,
options: RetryOptions = {}
): Promise<T> {
const {
maxAttempts = 3,
initialDelay = 1000,
maxDelay = 30000,
backoffMultiplier = 2,
jitter = true,
retryCondition = () => true
} = options;
let lastError: any;
let delay = initialDelay;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
return await fn();
} catch (error) {
lastError = error;
// Don't retry circuit open errors
if (error instanceof CircuitOpenError) {
throw error;
}
// Check if we should retry
if (!retryCondition(error)) {
throw error;
}
// Last attempt - throw
if (attempt === maxAttempts) {
log.error(`All ${maxAttempts} retry attempts failed`, { error });
throw error;
}
// Calculate delay with optional jitter
let waitTime = Math.min(delay, maxDelay);
if (jitter) {
waitTime = waitTime * (0.5 + Math.random());
}
log.warn(`Attempt ${attempt}/${maxAttempts} failed, retrying in ${Math.round(waitTime)}ms`, {
error: error instanceof Error ? error.message : String(error)
});
await sleep(waitTime);
delay *= backoffMultiplier;
}
}
throw lastError;
}
// ═══════════════════════════════════════════════════════════════════════════
// TIMEOUT WRAPPER
// ═══════════════════════════════════════════════════════════════════════════
export class TimeoutError extends Error {
constructor(message: string) {
super(message);
this.name = 'TimeoutError';
}
}
export async function withTimeout<T>(
fn: () => Promise<T>,
timeoutMs: number,
operation = 'Operation'
): Promise<T> {
return Promise.race([
fn(),
new Promise<never>((_, reject) =>
setTimeout(
() => reject(new TimeoutError(`${operation} timed out after ${timeoutMs}ms`)),
timeoutMs
)
)
]);
}
// ═══════════════════════════════════════════════════════════════════════════
// COMBINED: RESILIENT CALL
// ═══════════════════════════════════════════════════════════════════════════
export interface ResilientCallOptions extends RetryOptions {
timeout?: number;
circuitBreaker?: CircuitBreaker;
}
/**
* Execute a function with circuit breaker, retry, and timeout protection
*/
export async function resilientCall<T>(
fn: () => Promise<T>,
options: ResilientCallOptions = {}
): Promise<T> {
const { timeout, circuitBreaker, ...retryOptions } = options;
// Wrap with timeout if specified
let wrappedFn = fn;
if (timeout) {
wrappedFn = () => withTimeout(fn, timeout);
}
// Wrap with circuit breaker if specified
if (circuitBreaker) {
const originalFn = wrappedFn;
wrappedFn = () => circuitBreaker.execute(originalFn);
}
// Wrap with retry
return withRetry(wrappedFn, retryOptions);
}
// ═══════════════════════════════════════════════════════════════════════════
// CIRCUIT BREAKER REGISTRY
// ═══════════════════════════════════════════════════════════════════════════
const circuitBreakers = new Map<string, CircuitBreaker>();
/**
* Get or create a circuit breaker by name
*/
export function getCircuitBreaker(
name: string,
options?: Omit<CircuitBreakerOptions, 'name'>
): CircuitBreaker {
let cb = circuitBreakers.get(name);
if (!cb) {
cb = new CircuitBreaker({ name, ...options });
circuitBreakers.set(name, cb);
}
return cb;
}
/**
* Get status of all circuit breakers
*/
export function getAllCircuitBreakerStats() {
return Array.from(circuitBreakers.values()).map(cb => cb.getStats());
}
/**
* Reset all circuit breakers
*/
export function resetAllCircuitBreakers(): void {
circuitBreakers.forEach(cb => cb.reset());
}
// ═══════════════════════════════════════════════════════════════════════════
// UTILITY
// ═══════════════════════════════════════════════════════════════════════════
export function sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Check if an error is retryable (network errors, timeouts, 5xx)
*/
export function isRetryableError(error: any): boolean {
// Network errors
if (error.code === 'ECONNREFUSED' ||
error.code === 'ENOTFOUND' ||
error.code === 'ETIMEDOUT' ||
error.code === 'ECONNRESET') {
return true;
}
// Timeout errors
if (error instanceof TimeoutError) {
return true;
}
// HTTP 5xx errors
if (error.response?.status >= 500) {
return true;
}
// HTTP 429 (rate limited)
if (error.response?.status === 429) {
return true;
}
return false;
}