Update agent/server.py

agent/server.py

@@ -11,11 +11,12 @@ import uuid
 import json
 import traceback
 from typing import Optional, Callable
-from collections import deque
+from collections import deque, OrderedDict
 import time
 import math
 
 from fastapi import FastAPI
+from fastapi import Response
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
@@ -40,6 +41,41 @@ from tools.constraints_tool import apply_constraints
 
 os.environ.setdefault("HF_HOME", "/home/user/.cache/huggingface")
 
+import hashlib
+
+# ---------------------------
+# Simple in-memory TTL + LRU cache for responses
+# ---------------------------
+_CACHE_MAX_ITEMS = int(os.getenv("RECO_CACHE_MAX_ITEMS", "500"))
+_CACHE_TTL_SECONDS = int(os.getenv("RECO_CACHE_TTL_SECONDS", str(24 * 3600)))
+
+_reco_cache: OrderedDict[str, tuple[float, dict]] = OrderedDict()
+
+def _normalize_query(q: str) -> str:
+    return " ".join((q or "").lower().split())
+
+def _cache_key(query: str, llm_model: str | None, verbose: bool, endpoint: str) -> str:
+    model = (llm_model or os.getenv("LLM_MODEL", "") or "default").strip().lower()
+    raw = f"ep={endpoint}|q={_normalize_query(query)}|m={model}|v={int(verbose)}"
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+def _cache_get(key: str):
+    item = _reco_cache.get(key)
+    if not item:
+        return None
+    expires_at, value = item
+    if time.time() > expires_at:
+        _reco_cache.pop(key, None)
+        return None
+    _reco_cache.move_to_end(key)  # LRU refresh
+    return value
+
+def _cache_set(key: str, value: dict):
+    _reco_cache[key] = (time.time() + _CACHE_TTL_SECONDS, value)
+    _reco_cache.move_to_end(key)
+    while len(_reco_cache) > _CACHE_MAX_ITEMS:
+        _reco_cache.popitem(last=False)
+
 class ChatRequest(BaseModel):
     query: str
     clarification_answer: Optional[str] = None
@@ -306,14 +342,27 @@ def chat(req: ChatRequest):
 
 
 @app.post("/recommend")
-def recommend(req: RecommendRequest):
+def recommend(req: RecommendRequest, response: Response):
     if not _allow_request():
         return {"error": "rate limit exceeded"}
-    final_results, summary, debug_payload = _run_pipeline(req.query, verbose=req.verbose, llm_model=req.llm_model)
+
+    key = _cache_key(req.query, req.llm_model, req.verbose, endpoint="/recommend")
+    cached = _cache_get(key)
+    if cached is not None:
+        response.headers["X-Cache"] = "HIT"
+        return cached
+
+    response.headers["X-Cache"] = "MISS"
+
+    final_results, summary, debug_payload = _run_pipeline(
+        req.query, verbose=req.verbose, llm_model=req.llm_model
+    )
     resp = {"recommended_assessments": final_results}
     if req.verbose:
         resp["debug"] = _sanitize_debug(debug_payload)
         resp["summary"] = summary
+
+    _cache_set(key, resp)
     return resp