feat: implement caching, rate limiter, and security headers (CSP) for decoupled deploy

netlify.toml

@@ -2,3 +2,13 @@
   base = "client"
   command = "npm run build"
   publish = "dist"
+
+[[headers]]
+  for = "/*"
+  [headers.values]
+    Content-Security-Policy = "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https://* http://localhost:*; connect-src 'self' https://crambrodev-winx-prinx-api.hf.space ws://localhost:* http://localhost:* https://*; frame-ancestors 'none';"
+    X-Frame-Options = "DENY"
+    X-Content-Type-Options = "nosniff"
+    Referrer-Policy = "strict-origin-when-cross-origin"
+    Permissions-Policy = "geolocation=(), camera=(), microphone=()"
+    Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload"

server/cache.js

@@ -0,0 +1,260 @@
+/**
+ * production-ready In-Memory Caching and Rate-Limiting Subsystem
+ * Designed to protect the Supabase DB and Express server from spikes in traffic
+ */
+
+import dotenv from 'dotenv';
+dotenv.config();
+
+// Configuration flags
+const IS_CACHE_ENABLED = process.env.CACHE_ENABLED !== 'false';
+const DEFAULT_TTL_MS = (parseInt(process.env.CACHE_DEFAULT_TTL, 10) || 15) * 1000;
+
+const RATE_LIMIT_MAX = parseInt(process.env.RATE_LIMIT_MAX_REQUESTS, 10) || 60;
+const RATE_LIMIT_WINDOW_MS = parseInt(process.env.RATE_LIMIT_WINDOW_MS, 10) || 60000;
+
+// Metrics tracker
+const metrics = {
+  hits: 0,
+  misses: 0,
+  setCount: 0,
+  blockedRequests: 0,
+};
+
+// =========================================================================
+// CACHE MANAGER CLASS
+// =========================================================================
+class CacheManager {
+  constructor() {
+    this.store = new Map();
+    
+    // Background garbage collection sweep every 30 seconds
+    this.gcTimer = setInterval(() => {
+      this.evictExpired();
+    }, 30000).unref(); // .unref() allows Node process to exit cleanly if running tests
+  }
+
+  set(key, value, ttlMs = DEFAULT_TTL_MS) {
+    if (!IS_CACHE_ENABLED) return;
+    
+    const now = Date.now();
+    const expiresAt = now + ttlMs;
+    
+    this.store.set(key, {
+      value,
+      expiresAt,
+    });
+    metrics.setCount++;
+  }
+
+  get(key) {
+    if (!IS_CACHE_ENABLED) return null;
+    
+    const entry = this.store.get(key);
+    if (!entry) return null;
+    
+    const now = Date.now();
+    if (now > entry.expiresAt) {
+      this.store.delete(key);
+      return null;
+    }
+    
+    return entry;
+  }
+
+  del(key) {
+    return this.store.delete(key);
+  }
+
+  flush() {
+    this.store.clear();
+    console.log('[CacheManager] Cache flushed successfully.');
+  }
+
+  evictExpired() {
+    const now = Date.now();
+    let evictedCount = 0;
+    
+    for (const [key, entry] of this.store.entries()) {
+      if (now > entry.expiresAt) {
+        this.store.delete(key);
+        evictedCount++;
+      }
+    }
+    
+    if (evictedCount > 0) {
+      console.log(`[CacheManager GC] Swept and evicted ${evictedCount} expired cache keys.`);
+    }
+  }
+
+  getStats() {
+    const totalKeys = this.store.size;
+    const totalRequests = metrics.hits + metrics.misses;
+    const hitRatio = totalRequests > 0 ? ((metrics.hits / totalRequests) * 100).toFixed(1) + '%' : '0%';
+    
+    return {
+      enabled: IS_CACHE_ENABLED,
+      totalKeys,
+      hits: metrics.hits,
+      misses: metrics.misses,
+      hitRatio,
+      setCount: metrics.setCount,
+      blockedRequests: metrics.blockedRequests,
+    };
+  }
+}
+
+export const cache = new CacheManager();
+
+// =========================================================================
+// RATE LIMITER CLASS
+// =========================================================================
+class RateLimiter {
+  constructor() {
+    this.clients = new Map();
+    
+    // Background sweeper to clear expired rate-limit buckets every 30 seconds
+    this.gcTimer = setInterval(() => {
+      this.evictExpired();
+    }, 30000).unref();
+  }
+
+  /**
+   * Check if client IP is within limits
+   * @returns {object} { allowed: boolean, remaining: number, limit: number, resetTime: number }
+   */
+  check(ip) {
+    const now = Date.now();
+    let client = this.clients.get(ip);
+    
+    if (!client || now > client.resetTime) {
+      // Create new window
+      const resetTime = now + RATE_LIMIT_WINDOW_MS;
+      client = {
+        hits: 1,
+        resetTime,
+      };
+      this.clients.set(ip, client);
+      return {
+        allowed: true,
+        remaining: RATE_LIMIT_MAX - 1,
+        limit: RATE_LIMIT_MAX,
+        resetTime,
+      };
+    }
+    
+    client.hits++;
+    const remaining = Math.max(0, RATE_LIMIT_MAX - client.hits);
+    const allowed = client.hits <= RATE_LIMIT_MAX;
+    
+    return {
+      allowed,
+      remaining,
+      limit: RATE_LIMIT_MAX,
+      resetTime: client.resetTime,
+    };
+  }
+
+  evictExpired() {
+    const now = Date.now();
+    let count = 0;
+    
+    for (const [ip, data] of this.clients.entries()) {
+      if (now > data.resetTime) {
+        this.clients.delete(ip);
+        count++;
+      }
+    }
+    
+    if (count > 0) {
+      console.log(`[RateLimiter GC] Evicted ${count} expired rate-limit records.`);
+    }
+  }
+}
+
+const rateLimiter = new RateLimiter();
+
+// =========================================================================
+// EXPRESS MIDDLEWARE
+// =========================================================================
+/**
+ * Express Middleware protecting route with Rate-Limiting and serving/saving Cache
+ * @param {number} ttlSeconds Custom TTL override in seconds
+ */
+export function cacheMiddleware(ttlSeconds) {
+  const ttlMs = ttlSeconds ? ttlSeconds * 1000 : DEFAULT_TTL_MS;
+
+  return async (req, res, next) => {
+    // 1. Bypass check if cache is disabled globally
+    if (!IS_CACHE_ENABLED) {
+      return next();
+    }
+
+    // Extract real client IP (supports Express trust proxy)
+    const ip = req.ip || req.headers['x-forwarded-for'] || req.socket.remoteAddress;
+
+    // 2. Perform Rate Limiting
+    const limitStatus = rateLimiter.check(ip);
+    
+    // Set standard rate limit headers
+    res.setHeader('X-RateLimit-Limit', limitStatus.limit);
+    res.setHeader('X-RateLimit-Remaining', limitStatus.remaining);
+    res.setHeader('X-RateLimit-Reset', Math.ceil(limitStatus.resetTime / 1000));
+
+    if (!limitStatus.allowed) {
+      metrics.blockedRequests++;
+      const retryAfter = Math.ceil((limitStatus.resetTime - Date.now()) / 1000);
+      res.setHeader('Retry-After', retryAfter);
+      
+      console.warn(`[RateLimiter] Blocked IP: ${ip} for too many requests on path: ${req.path}`);
+      return res.status(429).json({
+        error: 'Too Many Requests',
+        message: 'Превышен лимит запросов к серверу. Пожалуйста, подождите.',
+        retryAfter,
+      });
+    }
+
+    // 3. Perform Cache Lookup
+    // Generate normalized cache key based on route path and alphabetically sorted query parameters
+    const sortedQueries = Object.keys(req.query)
+      .sort()
+      .map(k => `${k}=${req.query[k]}`)
+      .join('&');
+    
+    const cacheKey = sortedQueries ? `${req.path}?${sortedQueries}` : req.path;
+    const cachedEntry = cache.get(cacheKey);
+
+    if (cachedEntry) {
+      // Cache Hit
+      metrics.hits++;
+      const now = Date.now();
+      const remainingTtlSecs = Math.max(0, Math.ceil((cachedEntry.expiresAt - now) / 1000));
+      
+      res.setHeader('X-Cache', 'HIT');
+      res.setHeader('X-Cache-TTL-Remaining', remainingTtlSecs);
+      
+      // Serve cached JSON payload
+      return res.json(cachedEntry.value);
+    }
+
+    // Cache Miss
+    metrics.misses++;
+    res.setHeader('X-Cache', 'MISS');
+    res.setHeader('X-Cache-TTL-Remaining', Math.ceil(ttlMs / 1000));
+
+    // Intercept res.json to store the result in cache
+    const originalJson = res.json;
+    res.json = function (body) {
+      // Restore original res.json first to avoid recursive loops
+      res.json = originalJson;
+      
+      // Store in CacheManager
+      cache.set(cacheKey, body, ttlMs);
+      
+      // Call original response
+      return res.json(body);
+    };
+
+    next();
+  };
+}

server/migrate_to_supabase.js

@@ -0,0 +1,224 @@
+import { createClient } from '@supabase/supabase-js';
+import Database from 'better-sqlite3';
+import dotenv from 'dotenv';
+import path from 'path';
+
+dotenv.config();
+
+const supabaseUrl = process.env.SUPABASE_URL;
+const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_KEY;
+
+if (!supabaseUrl || !supabaseKey || supabaseUrl.includes('YOUR_') || supabaseKey.includes('YOUR_')) {
+  console.error('[Error] Supabase URL or Key is missing in .env!');
+  process.exit(1);
+}
+
+console.log('[Migration] Connecting to SQLite...');
+const sqliteDb = new Database('database.db');
+
+console.log('[Migration] Connecting to Supabase...');
+const supabase = createClient(supabaseUrl, supabaseKey);
+
+// Fetch valid stream IDs
+const streams = sqliteDb.prepare('SELECT id FROM streams').all();
+const validStreamIds = new Set(streams.map(s => s.id));
+console.log(`[Migration] Loaded ${validStreamIds.size} valid stream IDs from SQLite:`, Array.from(validStreamIds));
+
+async function migrateStreams() {
+  console.log('\n--- Migrating streams ---');
+  const streamsData = sqliteDb.prepare('SELECT * FROM streams').all();
+  if (streamsData.length === 0) {
+    console.log('No streams to migrate.');
+    return;
+  }
+  
+  console.log(`Found ${streamsData.length} streams. Inserting...`);
+  
+  const { error } = await supabase
+    .from('streams')
+    .upsert(streamsData);
+    
+  if (error) {
+    console.error('Error migrating streams:', error);
+  } else {
+    console.log('Successfully migrated streams!');
+  }
+}
+
+async function migrateSettings() {
+  console.log('\n--- Migrating settings ---');
+  const settings = sqliteDb.prepare('SELECT * FROM settings').all();
+  if (settings.length === 0) {
+    console.log('No settings to migrate.');
+    return;
+  }
+  
+  console.log(`Found ${settings.length} settings. Inserting...`);
+  const { error } = await supabase
+    .from('settings')
+    .upsert(settings);
+    
+  if (error) {
+    console.error('Error migrating settings:', error);
+  } else {
+    console.log('Successfully migrated settings!');
+  }
+}
+
+async function migrateModActions() {
+  console.log('\n--- Migrating mod_actions ---');
+  // Only migrate mod actions that reference valid stream IDs or have null stream_id
+  const modActions = sqliteDb.prepare('SELECT * FROM mod_actions').all();
+  if (modActions.length === 0) {
+    console.log('No mod actions to migrate.');
+    return;
+  }
+  
+  const filteredActions = modActions.filter(act => act.stream_id === null || validStreamIds.has(act.stream_id));
+  console.log(`Found ${modActions.length} mod actions. Migrating ${filteredActions.length} actions that have valid stream_id...`);
+  
+  if (filteredActions.length === 0) return;
+
+  const { error } = await supabase
+    .from('mod_actions')
+    .upsert(filteredActions);
+    
+  if (error) {
+    console.error('Error migrating mod actions:', error);
+  } else {
+    console.log('Successfully migrated mod actions!');
+  }
+}
+
+async function migrateMessages() {
+  console.log('\n--- Migrating messages ---');
+  const total = sqliteDb.prepare('SELECT count(*) as c FROM messages').get().c;
+  if (total === 0) {
+    console.log('No messages to migrate.');
+    return;
+  }
+  
+  console.log(`Found ${total} messages to migrate.`);
+  const batchSize = 1000;
+  let migratedCount = 0;
+  
+  for (let offset = 0; offset < total; offset += batchSize) {
+    const batch = sqliteDb.prepare('SELECT * FROM messages LIMIT ? OFFSET ?').all(batchSize, offset);
+    
+    // Filter messages that have valid stream_id
+    const filteredBatch = batch.filter(msg => msg.stream_id === null || validStreamIds.has(msg.stream_id));
+    
+    if (filteredBatch.length === 0) continue;
+
+    // Map SQLite types to Postgres types (booleans)
+    const mappedBatch = filteredBatch.map(msg => ({
+      id: msg.id,
+      stream_id: msg.stream_id,
+      username: msg.username,
+      display_name: msg.display_name,
+      message: msg.message,
+      timestamp: msg.timestamp,
+      is_streamer: !!msg.is_streamer,
+      is_mod: !!msg.is_mod,
+      is_sub: !!msg.is_sub
+    }));
+    
+    const { error } = await supabase
+      .from('messages')
+      .upsert(mappedBatch);
+      
+    if (error) {
+      console.error(`Error migrating messages batch at offset ${offset}:`, error);
+      console.log('Retrying...');
+      await new Promise(r => setTimeout(r, 1000));
+      const { error: retryError } = await supabase.from('messages').upsert(mappedBatch);
+      if (retryError) {
+        console.error('Retry failed, skipping batch:', retryError);
+      } else {
+        migratedCount += mappedBatch.length;
+      }
+    } else {
+      migratedCount += mappedBatch.length;
+    }
+    
+    if (offset % 5000 === 0 || offset + batchSize >= total) {
+      console.log(`[Messages Progress] Migrated ${Math.min(offset + batchSize, total)} / ${total} messages`);
+    }
+  }
+  
+  console.log(`Successfully migrated messages (inserted ${migratedCount} rows)!`);
+}
+
+async function migrateVoiceWords() {
+  console.log('\n--- Migrating voice_words ---');
+  const total = sqliteDb.prepare('SELECT count(*) as c FROM voice_words').get().c;
+  if (total === 0) {
+    console.log('No voice words to migrate.');
+    return;
+  }
+  
+  console.log(`Found ${total} voice words to migrate.`);
+  const batchSize = 3000;
+  let migratedCount = 0;
+  
+  for (let offset = 0; offset < total; offset += batchSize) {
+    const batch = sqliteDb.prepare('SELECT * FROM voice_words LIMIT ? OFFSET ?').all(batchSize, offset);
+    
+    // Filter voice words that have valid stream_id
+    const filteredBatch = batch.filter(vw => vw.stream_id === null || validStreamIds.has(vw.stream_id));
+    
+    if (filteredBatch.length === 0) continue;
+
+    const { error } = await supabase
+      .from('voice_words')
+      .upsert(filteredBatch);
+      
+    if (error) {
+      console.error(`Error migrating voice words batch at offset ${offset}:`, error);
+      console.log('Retrying...');
+      await new Promise(r => setTimeout(r, 1000));
+      const { error: retryError } = await supabase.from('voice_words').upsert(filteredBatch);
+      if (retryError) {
+        console.error('Retry failed, skipping batch:', retryError);
+      } else {
+        migratedCount += filteredBatch.length;
+      }
+    } else {
+      migratedCount += filteredBatch.length;
+    }
+    
+    if (offset % 30000 === 0 || offset + batchSize >= total) {
+      console.log(`[Voice Words Progress] Migrated ${Math.min(offset + batchSize, total)} / ${total} voice words`);
+    }
+  }
+  
+  console.log(`Successfully migrated voice words (inserted ${migratedCount} rows)!`);
+}
+
+async function startMigration() {
+  console.log('=== Starting database migration from SQLite to Supabase ===');
+  const startTime = Date.now();
+  
+  try {
+    await migrateStreams();
+    await migrateSettings();
+    await migrateModActions();
+    await migrateMessages();
+    await migrateVoiceWords();
+    
+    const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+    console.log(`\n=== Migration completed successfully in ${duration}s! ===`);
+    console.log('\nIMPORTANT: Now run the following SQL query in your Supabase SQL Editor to reset auto-increment sequences:');
+    console.log(`
+SELECT setval(pg_get_serial_sequence('streams', 'id'), coalesce(max(id), 1)) FROM streams;
+SELECT setval(pg_get_serial_sequence('voice_words', 'id'), coalesce(max(id), 1)) FROM voice_words;
+SELECT setval(pg_get_serial_sequence('mod_actions', 'id'), coalesce(max(id), 1)) FROM mod_actions;
+    `);
+  } catch (err) {
+    console.error('Migration failed:', err);
+  } finally {
+    sqliteDb.close();
+  }
+}
+
+startMigration();

server/server.js

@@ -33,10 +33,12 @@ import {
   cleanupGhostStreams
 } from './db.js';
 // import { initializeEventSub } from './eventsub.js';
+import { cache, cacheMiddleware } from './cache.js';
 
 dotenv.config();
 
 const app = express();
+app.set('trust proxy', true); // Safe trust proxy behind Cloudflare and Hugging Face proxies
 const PORT = process.env.PORT || 3000;
 
 // Configure CORS to allow frontend connections
@@ -259,6 +261,7 @@ app.post('/api/streams/reset-backfill', requireModeratorRole, async (req, res) =
   }
   try {
     await resetStreamBackfill(parseInt(streamId), mode || 'gap_fill');
+    cache.flush();
     res.json({ success: true });
   } catch (err) {
     res.status(500).json({ error: err.message });
@@ -273,6 +276,7 @@ app.delete('/api/streams/:id', requireAdminRole, async (req, res) => {
   }
   try {
     await deleteStream(streamId);
+    cache.flush();
     res.json({ success: true });
   } catch (err) {
     res.status(500).json({ error: err.message });
@@ -291,6 +295,7 @@ app.put('/api/streams/:id', requireAdminRole, async (req, res) => {
   }
   try {
     await updateStreamMetadata(streamId, title, category || '');
+    cache.flush();
     res.json({ success: true });
   } catch (err) {
     res.status(500).json({ error: err.message });
@@ -301,6 +306,7 @@ app.put('/api/streams/:id', requireAdminRole, async (req, res) => {
 app.post('/api/admin/cleanup', requireAdminRole, async (req, res) => {
   try {
     const deletedCount = await cleanupGhostStreams();
+    cache.flush();
     res.json({ success: true, count: deletedCount });
   } catch (err) {
     res.status(500).json({ error: err.message });
@@ -311,7 +317,8 @@ app.post('/api/admin/cleanup', requireAdminRole, async (req, res) => {
 app.get('/api/admin/stats', requireAdminRole, async (req, res) => {
   try {
     const stats = await getSystemStats();
-    res.json({ success: true, stats });
+    const cacheStats = cache.getStats();
+    res.json({ success: true, stats, cacheStats });
   } catch (err) {
     res.status(500).json({ error: err.message });
   }
@@ -476,13 +483,13 @@ app.get('/api/auth/logout', (req, res) => {
 // =========================================================================
 
 // Get list of streams
-app.get('/api/streams', async (req, res) => {
+app.get('/api/streams', cacheMiddleware(), async (req, res) => {
   const streams = await getStreamsList();
   res.json(streams);
 });
 
 // Get top active chatters
-app.get('/api/stats/chatters', async (req, res) => {
+app.get('/api/stats/chatters', cacheMiddleware(), async (req, res) => {
   const streamId = req.query.stream_id ? parseInt(req.query.stream_id) : null;
   const limit = req.query.limit ? parseInt(req.query.limit) : 50;
   
@@ -516,7 +523,7 @@ const STOP_WORDS = new Set([
 ]);
 
 // Get word frequencies (Voice vs Chat messages from streamer)
-app.get('/api/stats/words', async (req, res) => {
+app.get('/api/stats/words', cacheMiddleware(), async (req, res) => {
   const streamId = req.query.stream_id ? parseInt(req.query.stream_id) : null;
   const limit = req.query.limit ? parseInt(req.query.limit) : 50;
   const type = req.query.type || 'voice'; // 'voice' or 'chat'
@@ -610,7 +617,7 @@ app.get('/api/stats/words', async (req, res) => {
 });
 
 // Get chat activity (messages count grouped by 5-minute intervals)
-app.get('/api/stats/activity', async (req, res) => {
+app.get('/api/stats/activity', cacheMiddleware(), async (req, res) => {
   const streamId = req.query.stream_id ? parseInt(req.query.stream_id) : null;
 
   if (!streamId) {
@@ -688,7 +695,7 @@ app.get('/api/stats/moderators/summary', requireModeratorRole, async (req, res)
 });
 
 // GET aggregated profiles with scores for moderators
-app.get('/api/stats/moderators/profiles', async (req, res) => {
+app.get('/api/stats/moderators/profiles', cacheMiddleware(), async (req, res) => {
   const streamId = req.query.stream_id ? parseInt(req.query.stream_id) : null;
   const profiles = await getModeratorProfilesData(streamId);
   res.json(profiles);