perf: reduce per-query latency — shared model, single encode, float32 LLM

- Add modules/shared.py: single SentenceTransformer singleton loaded once
- HybridRetriever and DriftDetector now share the same model instance
instead of each loading a separate copy (saves ~90MB RAM + load time)
- Encode query embedding once in process_query and pass to both
analyze_drift() and search() — eliminates redundant encode call
- Switch LLM to torch.float32 (bfloat16 has no CPU hardware acceleration)
- Reduce max_new_tokens 120→80 for faster token generation

app.py

@@ -11,6 +11,7 @@ import sys
 import gradio as gr
 import plotly.graph_objects as go
 from modules.data_simulation import generate_catalog, get_scenarios
+from modules.shared import get_embedding_model
 from modules.retrieval import HybridRetriever
 from modules.drift import DriftDetector
 from modules.adaptation import Adapter
@@ -229,11 +230,14 @@ def process_query(query: str, history: list):
 
     logger.info("Processing query: %r", query)
 
+    # Encode query once — shared by drift detection and retrieval
+    query_emb = get_embedding_model().encode([query], show_progress_bar=False)[0]
+
     # 1. Measure drift
-    drift_state, scores = detector.analyze_drift(query)
+    drift_state, scores = detector.analyze_drift(query, query_emb=query_emb)
 
     # 2. Retrieve products (hybrid: price-filter + semantic)
-    retrieved = retriever.search(query, top_k=4)
+    retrieved = retriever.search(query, top_k=4, query_emb=query_emb)
 
     # 3. Adapt system prompt
     system_prompt = adapter.adapt_prompt(drift_state)

modules/drift.py

@@ -16,19 +16,10 @@ from dataclasses import dataclass, field
 from typing import Any
 
 import numpy as np
-from sentence_transformers import SentenceTransformer
-
-logger = logging.getLogger(__name__)
-
-# Use shared model instance across retriever & drift detector
-_shared_model: SentenceTransformer | None = None
 
+from modules.shared import get_embedding_model
 
-def _get_model() -> SentenceTransformer:
-    global _shared_model
-    if _shared_model is None:
-        _shared_model = SentenceTransformer("all-MiniLM-L6-v2")
-    return _shared_model
+logger = logging.getLogger(__name__)
 
 
 @dataclass
@@ -57,7 +48,7 @@ class DriftDetector:
     _concept_embs: dict[str, Any] = field(default_factory=dict, repr=False)
 
     def __post_init__(self) -> None:
-        model = _get_model()
+        model = get_embedding_model()
         # Multiple anchor phrases per concept → averaged embedding for robustness
         concept_phrases = {
             "price_sensitive": [
@@ -95,13 +86,17 @@ class DriftDetector:
                 self._ewma[c] = 0.15
     # ── Public API ──────────────────────────────────────────────────────────
 
-    def analyze_drift(self, query: str) -> tuple[str, dict[str, float]]:
+    def analyze_drift(
+        self, query: str, query_emb=None
+    ) -> tuple[str, dict[str, float]]:
         """
         Score *query* against all concept anchors and return
         ``(dominant_concept, raw_scores)``.
+
+        Pass *query_emb* to skip re-encoding when the caller already has it.
         """
-        model = _get_model()
-        query_emb = model.encode([query], show_progress_bar=False)[0]
+        if query_emb is None:
+            query_emb = get_embedding_model().encode([query], show_progress_bar=False)[0]
 
         raw_scores: dict[str, float] = {}
         for concept, ref_emb in self._concept_embs.items():

modules/llm.py

@@ -30,7 +30,7 @@ def _get_pipeline():
             "text-generation",
             model="Qwen/Qwen2.5-0.5B-Instruct",
             device="cpu",
-            torch_dtype=torch.bfloat16,
+            torch_dtype=torch.float32,
         )
         logger.info("Model loaded in %.1fs", time.time() - t0)
     return _generator
@@ -81,7 +81,7 @@ def generate_response(
         gen = _get_pipeline()
         result = gen(
             messages,
-            max_new_tokens=120,
+            max_new_tokens=80,
             do_sample=False,
             return_full_text=False,
         )

modules/retrieval.py

@@ -13,7 +13,8 @@ import re
 from typing import Any
 
 import numpy as np
-from sentence_transformers import SentenceTransformer
+
+from modules.shared import get_embedding_model
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +24,7 @@ class HybridRetriever:
 
     def __init__(self, catalog: list[dict]) -> None:
         self.catalog = catalog
-        self.model = SentenceTransformer("all-MiniLM-L6-v2")
+        self.model = get_embedding_model()
 
         # Build rich embedding texts that capture all searchable facets
         texts = [
@@ -47,6 +48,7 @@ class HybridRetriever:
         query: str,
         top_k: int = 4,
         category_filter: str | None = None,
+        query_emb=None,
     ) -> list[dict[str, Any]]:
         """
         Retrieve top-k products for *query*.
@@ -56,6 +58,8 @@ class HybridRetriever:
         2. Pre-filter catalog by price / category if applicable.
         3. Rank remaining items by cosine similarity.
         4. Return top-k with scores.
+
+        Pass *query_emb* to skip re-encoding when the caller already has it.
         """
         price_cap = self._extract_price_cap(query)
         cat_hint = category_filter or self._extract_category_hint(query)
@@ -64,7 +68,8 @@ class HybridRetriever:
         candidate_indices = self._prefilter(price_cap, cat_hint)
 
         # Stage 2 — semantic ranking over candidates
-        query_emb = self.model.encode([query], show_progress_bar=False)[0]
+        if query_emb is None:
+            query_emb = self.model.encode([query], show_progress_bar=False)[0]
         query_norm = np.linalg.norm(query_emb)
 
         if len(candidate_indices) == 0:

modules/shared.py

@@ -0,0 +1,20 @@
+"""Shared SentenceTransformer singleton — loaded once, used everywhere."""
+
+from __future__ import annotations
+
+import logging
+
+from sentence_transformers import SentenceTransformer
+
+logger = logging.getLogger(__name__)
+
+_model: SentenceTransformer | None = None
+
+
+def get_embedding_model() -> SentenceTransformer:
+    global _model
+    if _model is None:
+        logger.info("Loading SentenceTransformer (all-MiniLM-L6-v2)…")
+        _model = SentenceTransformer("all-MiniLM-L6-v2")
+        logger.info("SentenceTransformer ready.")
+    return _model