Enable llama.cpp KV prompt cache (512 MB): ~21% off study-plan time

main.py

@@ -21,6 +21,7 @@ from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel, Field
 from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
+from llama_cpp.llama_cache import LlamaRAMCache
 from llama_cpp import llama_chat_format
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(levelname)s  %(message)s")
@@ -50,6 +51,10 @@ _USABLE = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else
 N_THREADS          = int(os.getenv("N_THREADS", str(_USABLE)))
 N_GPU_LAYERS       = int(os.getenv("N_GPU_LAYERS", "0"))
 DEFAULT_MAX_TOKENS = int(os.getenv("DEFAULT_MAX_TOKENS", "512"))
+# Prompt (KV) cache. Without it every call re-prefills the whole system prompt,
+# which is NOT cheap on 2 vCPU: measured 20.8s -> 8.5s per call with the study
+# plan's rotating daily/weekly/monthly prompts, ~21% off a full plan.
+PROMPT_CACHE_MB    = int(os.getenv("PROMPT_CACHE_MB", "512"))   # 0 disables
 
 # Current model state (mutable at runtime via /admin/load).
 _llm: Optional[Llama] = None
@@ -81,6 +86,9 @@ def _download_and_load(cfg: Dict[str, str]) -> None:
         if cfg["chat_format"]:
             kwargs["chat_format"] = cfg["chat_format"]
         new_llm = Llama(**kwargs)
+        if PROMPT_CACHE_MB > 0:
+            new_llm.set_cache(LlamaRAMCache(capacity_bytes=PROMPT_CACHE_MB * 1024 * 1024))
+            log.info("Prompt cache enabled (%d MB).", PROMPT_CACHE_MB)
         with _llm_lock:                      # swap atomically; old model is freed
             _llm = new_llm
             _cfg = dict(cfg)