Spaces:
Runtime error
Runtime error
File size: 2,337 Bytes
86042ad |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 |
import log from "encore.dev/log";
import { CacheEntry } from "./types";
// Simple in-memory cache for LLM responses
class SimpleCache<T> {
private cache: Map<string, CacheEntry<T>> = new Map();
private readonly ttl: number;
private readonly maxEntries: number;
constructor(ttlSeconds: number = 300, maxEntries: number = 100) {
this.ttl = ttlSeconds * 1000;
this.maxEntries = maxEntries;
// Clean up expired entries every minute
setInterval(() => this.cleanup(), 60000);
}
// Generate cache key from request parameters
generateKey(params: Record<string, any>): string {
return JSON.stringify(params);
}
// Get cached value if it exists and is not expired
get(key: string): T | null {
const entry = this.cache.get(key);
if (!entry) {
return null;
}
if (Date.now() > entry.expiresAt) {
this.cache.delete(key);
return null;
}
log.info("Cache hit", { key: key.substring(0, 50) });
return entry.data;
}
set(key: string, value: T): void {
if (this.cache.size >= this.maxEntries) {
const firstKey = this.cache.keys().next().value as string | undefined;
if (firstKey) {
this.cache.delete(firstKey);
}
}
this.cache.set(key, {
data: value,
timestamp: Date.now(),
expiresAt: Date.now() + this.ttl,
});
log.info("Cache set", {
key: key.substring(0, 50),
size: this.cache.size
});
}
// Remove expired entries
private cleanup(): void {
const now = Date.now();
let removed = 0;
for (const [key, entry] of this.cache.entries()) {
if (now > entry.expiresAt) {
this.cache.delete(key);
removed++;
}
}
if (removed > 0) {
log.info("Cache cleanup", { removed, remaining: this.cache.size });
}
}
// Clear all cache entries
clear(): void {
this.cache.clear();
log.info("Cache cleared");
}
// Get cache statistics
getStats() {
return {
size: this.cache.size,
maxEntries: this.maxEntries,
ttl: this.ttl / 1000,
};
}
}
// Export singleton instances for different cache types
export const chatCache = new SimpleCache<string>(300, 100);
export const ragCache = new SimpleCache<string>(600, 50);
export const analysisCache = new SimpleCache<string>(900, 30);
|