feat: implement LRU cache eviction policy to prevent memory leaks

server/.env.example

@@ -24,3 +24,9 @@ ADMIN_USERNAMES=crimbr1243
 # Security Key for Local Worker
 # The Local Worker must send this key in the 'x-api-key' header when posting logs
 API_KEY=a_long_secure_token_for_local_worker_to_server_sync_98765
+
+# Caching Configuration
+# CACHE_ENABLED=true
+# CACHE_DEFAULT_TTL=15
+# CACHE_MAX_KEYS=1000
+

server/cache.js

@@ -9,6 +9,7 @@ dotenv.config();
 // Configuration flags
 const IS_CACHE_ENABLED = process.env.CACHE_ENABLED !== 'false';
 const DEFAULT_TTL_MS = (parseInt(process.env.CACHE_DEFAULT_TTL, 10) || 15) * 1000;
+const CACHE_MAX_KEYS = parseInt(process.env.CACHE_MAX_KEYS, 10) || 1000;
 
 const RATE_LIMIT_MAX = parseInt(process.env.RATE_LIMIT_MAX_REQUESTS, 10) || 60;
 const RATE_LIMIT_WINDOW_MS = parseInt(process.env.RATE_LIMIT_WINDOW_MS, 10) || 60000;
@@ -40,6 +41,18 @@ class CacheManager {
     const now = Date.now();
     const expiresAt = now + ttlMs;
     
+    if (this.store.has(key)) {
+      // Delete existing to update its insertion order position (LRU)
+      this.store.delete(key);
+    } else if (this.store.size >= CACHE_MAX_KEYS) {
+      // Evict the least recently used (oldest) key
+      const oldestKey = this.store.keys().next().value;
+      if (oldestKey !== undefined) {
+        this.store.delete(oldestKey);
+        console.log(`[CacheManager LRU] Evicted oldest key: ${oldestKey}`);
+      }
+    }
+    
     this.store.set(key, {
       value,
       expiresAt,
@@ -53,12 +66,11 @@ class CacheManager {
     const entry = this.store.get(key);
     if (!entry) return null;
     
-    const now = Date.now();
-    if (now > entry.expiresAt) {
-      this.store.delete(key);
-      return null;
-    }
+    // Refresh insertion order position for LRU
+    this.store.delete(key);
+    this.store.set(key, entry);
     
+    // Return the entry even if expired. Freshness logic is handled by middleware
     return entry;
   }
 
@@ -76,7 +88,8 @@ class CacheManager {
     let evictedCount = 0;
     
     for (const [key, entry] of this.store.entries()) {
-      if (now > entry.expiresAt) {
+      // Keep stale data in memory for up to 24 hours so it can be served during heavy compute spikes
+      if (now > entry.expiresAt + 24 * 60 * 60 * 1000) {
         this.store.delete(key);
         evictedCount++;
       }
@@ -177,8 +190,11 @@ const rateLimiter = new RateLimiter();
 // =========================================================================
 // EXPRESS MIDDLEWARE
 // =========================================================================
+
+const pendingRequests = new Map();
+
 /**
- * Express Middleware protecting route with Rate-Limiting and serving/saving Cache
+ * Express Middleware protecting route with Rate-Limiting, Coalescing, and Stale-While-Revalidate Cache
  * @param {number} ttlSeconds Custom TTL override in seconds
  */
 export function cacheMiddleware(ttlSeconds) {
@@ -215,43 +231,80 @@ export function cacheMiddleware(ttlSeconds) {
     }
 
     // 3. Perform Cache Lookup
-    // Generate normalized cache key based on route path and alphabetically sorted query parameters
     const sortedQueries = Object.keys(req.query)
       .sort()
       .map(k => `${k}=${req.query[k]}`)
       .join('&');
     
     const cacheKey = sortedQueries ? `${req.path}?${sortedQueries}` : req.path;
-    const cachedEntry = cache.get(cacheKey);
+    const entry = cache.get(cacheKey);
+    const now = Date.now();
 
-    if (cachedEntry) {
-      // Cache Hit
+    // CASE A: Cache is FRESH
+    if (entry && now <= entry.expiresAt) {
       metrics.hits++;
-      const now = Date.now();
-      const remainingTtlSecs = Math.max(0, Math.ceil((cachedEntry.expiresAt - now) / 1000));
-      
+      const remainingTtlSecs = Math.max(0, Math.ceil((entry.expiresAt - now) / 1000));
       res.setHeader('X-Cache', 'HIT');
       res.setHeader('X-Cache-TTL-Remaining', remainingTtlSecs);
-      
-      // Serve cached JSON payload
-      return res.json(cachedEntry.value);
+      return res.json(entry.value);
     }
 
-    // Cache Miss
+    // CASE B: Cache is STALE or MISS, and another request is already computing it
+    if (pendingRequests.has(cacheKey)) {
+      if (entry) {
+        // Stale-While-Revalidate: Return old data immediately to avoid waiting
+        metrics.hits++;
+        res.setHeader('X-Cache', 'STALE');
+        return res.json(entry.value);
+      } else {
+        // Request Coalescing: No old data exists, so wait for the pending request to finish
+        metrics.misses++;
+        res.setHeader('X-Cache', 'COALESCED');
+        try {
+          const value = await pendingRequests.get(cacheKey);
+          return res.json(value);
+        } catch (e) {
+          return res.status(500).json({ error: 'Internal Server Error computing cache' });
+        }
+      }
+    }
+
+    // CASE C: Cache is STALE or MISS, and WE are the first request to trigger compute
     metrics.misses++;
     res.setHeader('X-Cache', 'MISS');
-    res.setHeader('X-Cache-TTL-Remaining', Math.ceil(ttlMs / 1000));
 
-    // Intercept res.json to store the result in cache
+    // Create a pending Promise that others can wait on
+    let resolvePending;
+    let rejectPending;
+    const pendingPromise = new Promise((resolve, reject) => {
+      resolvePending = resolve;
+      rejectPending = reject;
+    });
+    pendingRequests.set(cacheKey, pendingPromise);
+
+    // Timeout safety net to prevent frozen promises if next() hangs forever
+    const safetyTimeout = setTimeout(() => {
+      if (pendingRequests.get(cacheKey) === pendingPromise) {
+        pendingRequests.delete(cacheKey);
+        rejectPending(new Error('Cache compute timeout'));
+      }
+    }, 5 * 60 * 1000);
+
+    // Intercept res.json to store the result
     const originalJson = res.json;
     res.json = function (body) {
-      // Restore original res.json first to avoid recursive loops
-      res.json = originalJson;
+      clearTimeout(safetyTimeout);
+      res.json = originalJson; // Restore
       
-      // Store in CacheManager
+      // Save fresh data
       cache.set(cacheKey, body, ttlMs);
       
-      // Call original response
+      // Resolve the promise to unblock anyone who was coalesced (waiting)
+      resolvePending(body);
+      if (pendingRequests.get(cacheKey) === pendingPromise) {
+        pendingRequests.delete(cacheKey);
+      }
+      
       return res.json(body);
     };