Create cache.py

services/cache.py

@@ -0,0 +1,24 @@
+# cache.py 
+from functools import lru_cache
+from typing import Tuple, Any
+
+# TODO explain howto use the cache
+class ResponseCache:
+    def __init__(self, cache_size: int = 1000):
+        self.cache_size = cache_size
+        self._initialize_cache()
+
+    def _initialize_cache(self):
+        @lru_cache(maxsize=self.cache_size)
+        def cached_response(prompt: str, config_hash: str) -> Tuple[str, float]:
+            pass
+        self.get_cached_response = cached_response
+
+    def cache_response(self, prompt: str, config: GenerationConfig, response: str, score: float) -> None:
+        config_hash = hash(str(config.__dict__))
+        self.get_cached_response(prompt, str(config_hash))
+        
+    def get_response(self, prompt: str, config: GenerationConfig) -> Optional[Tuple[str, float]]:
+        config_hash = hash(str(config.__dict__))
+        return self.get_cached_response(prompt, str(config_hash))
+