/** * Semantic Cache — Phase 9.1 * * Caches non-streaming LLM responses (temperature=0) to reduce cost and latency. * Two-tier: in-memory LRU (fast) + SQLite (persistent across restarts). * * Cache key = SHA-256(model + normalized messages + temperature + top_p) * Bypass: X-OmniRoute-No-Cache: true * * @module lib/semanticCache */ import crypto from "node:crypto"; import { LRUCache } from "./cacheLayer"; import { getDbInstance } from "./db/core"; // ─── Singleton ───────────────── let memoryCache; let stats = { hits: 0, misses: 0, tokensSaved: 0 }; function getMemoryCache() { if (!memoryCache) { memoryCache = new LRUCache({ maxSize: parseInt(process.env.SEMANTIC_CACHE_MAX_SIZE || "500", 10), defaultTTL: parseInt(process.env.SEMANTIC_CACHE_TTL_MS || "3600000", 10), // 1h }); } return memoryCache; } // ─── Signature Generation ───────────────── /** * Generate deterministic cache signature from request params. * @param {string} model * @param {Array} messages - Normalized messages array * @param {number} temperature * @param {number} topP * @returns {string} hex signature */ export function generateSignature(model, messages, temperature = 0, topP = 1) { const payload = JSON.stringify({ model, messages: normalizeMessages(messages), temperature, top_p: topP, }); return crypto.createHash("sha256").update(payload).digest("hex"); } /** * Normalize messages for consistent hashing. * Strips metadata, keeps only role + content. */ function normalizeMessages(messages) { if (!Array.isArray(messages)) return []; return messages.map((m) => ({ role: m.role || "user", content: typeof m.content === "string" ? m.content : JSON.stringify(m.content), })); } // ─── Cache Operations ───────────────── /** * Check if a cached response exists for the given signature. * Checks memory first, then SQLite. * @param {string} signature * @returns {object|null} Cached response or null */ export function getCachedResponse(signature) { // 1. Check memory cache const memResult = getMemoryCache().get(signature); if (memResult) { stats.hits++; stats.tokensSaved += memResult.tokensSaved || 0; return memResult.response; } // 2. Check SQLite try { const db = getDbInstance(); const row = db .prepare( "SELECT response, tokens_saved FROM semantic_cache WHERE signature = ? AND expires_at > datetime('now')" ) .get(signature); if (row) { const parsed = JSON.parse(row.response); // Promote to memory cache getMemoryCache().set(signature, { response: parsed, tokensSaved: row.tokens_saved, }); // Update hit count in DB db.prepare("UPDATE semantic_cache SET hit_count = hit_count + 1 WHERE signature = ?").run( signature ); stats.hits++; stats.tokensSaved += row.tokens_saved || 0; return parsed; } } catch { // DB not available — fail open } stats.misses++; return null; } /** * Store a response in cache. * @param {string} signature * @param {string} model * @param {object} response - The API response to cache * @param {number} tokensSaved - Estimated tokens saved * @param {number} [ttlMs] - TTL in ms (default: 1 hour) */ export function setCachedResponse(signature, model, response, tokensSaved = 0, ttlMs = 3600000) { const ttl = parseInt(process.env.SEMANTIC_CACHE_TTL_MS || String(ttlMs), 10); // 1. Memory cache getMemoryCache().set(signature, { response, tokensSaved }, ttl); // 2. SQLite try { const db = getDbInstance(); const id = crypto.randomUUID(); const promptHash = signature.slice(0, 16); const now = new Date().toISOString(); const expiresAt = new Date(Date.now() + ttl).toISOString(); db.prepare( `INSERT OR REPLACE INTO semantic_cache (id, signature, model, prompt_hash, response, tokens_saved, hit_count, created_at, expires_at) VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?)` ).run(id, signature, model, promptHash, JSON.stringify(response), tokensSaved, now, expiresAt); } catch { // DB write failed — cache still in memory } } // ─── Maintenance ───────────────── /** * Remove expired entries from SQLite. * @returns {number} Number of entries removed */ export function cleanExpiredEntries() { try { const db = getDbInstance(); const result = db .prepare("DELETE FROM semantic_cache WHERE expires_at <= datetime('now')") .run(); return result.changes; } catch { return 0; } } /** * Invalidate cache entries by model name. * Useful when a model is updated/changed and cached responses are stale. * @param {string} model - Model name to invalidate (exact match) * @returns {number} Number of entries removed */ export function invalidateByModel(model: string): number { getMemoryCache().clear(); // Memory cache doesn't track model; full clear try { const db = getDbInstance(); const result = db.prepare("DELETE FROM semantic_cache WHERE model = ?").run(model); return result.changes || 0; } catch { return 0; } } /** * Invalidate a single cache entry by its signature. * @param {string} signature - Cache signature to invalidate * @returns {boolean} Whether the entry was found and removed */ export function invalidateBySignature(signature: string): boolean { getMemoryCache().delete(signature); try { const db = getDbInstance(); const result = db.prepare("DELETE FROM semantic_cache WHERE signature = ?").run(signature); return (result.changes || 0) > 0; } catch { return false; } } /** * Invalidate entries older than a given age. * @param {number} maxAgeMs - Maximum age in milliseconds * @returns {number} Number of entries removed */ export function invalidateStale(maxAgeMs: number): number { getMemoryCache().clear(); try { const db = getDbInstance(); const cutoff = new Date(Date.now() - maxAgeMs).toISOString(); const result = db.prepare("DELETE FROM semantic_cache WHERE created_at < ?").run(cutoff); return result.changes || 0; } catch { return 0; } } // ── Auto-cleanup timer ── let _cleanupTimer: ReturnType | null = null; /** * Start periodic auto-cleanup of expired entries. * @param {number} intervalMs - Cleanup interval (default: 5 minutes) */ export function startAutoCleanup(intervalMs = 300_000): void { stopAutoCleanup(); _cleanupTimer = setInterval(() => { const removed = cleanExpiredEntries(); if (removed > 0) { console.log(`[SemanticCache] Auto-cleaned ${removed} expired entries`); } }, intervalMs); } /** * Stop periodic auto-cleanup. */ export function stopAutoCleanup(): void { if (_cleanupTimer) { clearInterval(_cleanupTimer); _cleanupTimer = null; } } /** * Clear all cache entries. */ export function clearCache() { getMemoryCache().clear(); try { const db = getDbInstance(); db.prepare("DELETE FROM semantic_cache").run(); } catch { // DB not available } stats = { hits: 0, misses: 0, tokensSaved: 0 }; } // ─── Stats ───────────────── /** * Get cache statistics. */ export function getCacheStats() { const memStats = getMemoryCache().getStats(); let dbSize = 0; try { const db = getDbInstance(); const row = db .prepare("SELECT COUNT(*) as count FROM semantic_cache WHERE expires_at > datetime('now')") .get(); dbSize = row?.count || 0; } catch { // DB not available } const total = stats.hits + stats.misses; return { memoryEntries: memStats.size, dbEntries: dbSize, hits: stats.hits, misses: stats.misses, hitRate: total > 0 ? ((stats.hits / total) * 100).toFixed(1) : "0.0", tokensSaved: stats.tokensSaved, }; } /** * Check if a request is cacheable. * Only non-streaming, deterministic (temperature=0) requests. */ export function isCacheable(body, headers) { if (headers?.get?.("x-omniroute-no-cache") === "true") return false; if (body.stream !== false) return false; if ((body.temperature ?? 0) !== 0) return false; return true; }