Update main.py

main.py

@@ -1,32 +1,29 @@
 """
-main.py — Pricelyst Shopping Advisor (Jessica Edition)
+main.py — Pricelyst Shopping Advisor (Jessica Edition - Grounded Data Version)
 
 ✅ Flask API
-✅ Firebase Admin persistence (service account JSON via env var)
-✅ Gemini via NEW google-genai SDK (text + multimodal + JSON Mode)
-✅ Product intelligence from Pricelyst API
-✅ Graceful conversational handling (Backwards Compatible)
-✅ Call briefing (Zim Essentials Injection)
-✅ Post-call Shopping Plan Generation (PDF-ready)
-
-ENV VARS YOU NEED:
+✅ Firebase Admin persistence
+✅ Gemini via google-genai SDK
+✅ RAG (Retrieval Augmented Generation) for Shopping Plans
+✅ Real Pricing Logic (No Hallucinations)
+✅ Backwards Compatible with React Client
+
+ENV VARS:
 - GOOGLE_API_KEY=...
-- FIREBASE='{"type":"service_account", ...}'   # full JSON string
-- PRICE_API_BASE=https://api.pricelyst.co.zw   # optional
-- GEMINI_MODEL=gemini-2.0-flash                # optional
-- PORT=5000                                    # optional
+- FIREBASE='{"type":"service_account", ...}'
+- PRICE_API_BASE=https://api.pricelyst.co.zw
+- GEMINI_MODEL=gemini-2.0-flash
+- PORT=5000
 """
 
 import os
 import re
 import json
 import time
-import math
-import uuid
 import base64
 import logging
 from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional
 
 import requests
 import pandas as pd
@@ -43,8 +40,6 @@ logger = logging.getLogger("pricelyst-advisor")
 
 # ––––– Gemini (NEW SDK) –––––
 
-# pip install google-genai
-
 try:
     from google import genai
     from google.genai import types
@@ -65,22 +60,19 @@ if genai and GOOGLE_API_KEY:
 
 # ––––– Firebase Admin –––––
 
-# pip install firebase-admin
-
 import firebase_admin
 from firebase_admin import credentials, firestore
 
 FIREBASE_ENV = os.environ.get("FIREBASE", "")
 
 def init_firestore_from_env() -> firestore.Client:
-    # 1. Check if already initialized
     if firebase_admin._apps:
         return firestore.client()
 
-    # 2. Check for Creds
     if not FIREBASE_ENV:
-        logger.critical("FIREBASE env var missing. Persistence will fail.")
-        raise RuntimeError("FIREBASE env var missing. Provide full service account JSON string.")
+        # Fallback for local dev if needed, or raise error
+        logger.warning("FIREBASE env var missing. Persistence disabled.")
+        return None
 
     try:
         sa_info = json.loads(FIREBASE_ENV)
@@ -90,18 +82,14 @@ def init_firestore_from_env() -> firestore.Client:
         return firestore.client()
     except Exception as e:
         logger.critical("Failed to initialize Firebase: %s", e)
-        raise e
+        return None
 
-try:
-    db = init_firestore_from_env()
-except Exception as e:
-    logger.error("DB Init failed: %s", e)
-    db = None
+db = init_firestore_from_env()
 
 # ––––– External API (Pricelyst) –––––
 
 PRICE_API_BASE = os.environ.get("PRICE_API_BASE", "https://api.pricelyst.co.zw").rstrip("/")
-HTTP_TIMEOUT = 20
+HTTP_TIMEOUT = 25
 
 # ––––– Flask –––––
 
@@ -110,42 +98,38 @@ CORS(app)
 
 # ––––– In-memory product cache –––––
 
-PRODUCT_CACHE_TTL_SEC = 60 * 10  # 10 minutes
+PRODUCT_CACHE_TTL_SEC = 60 * 15  # 15 minutes
 _product_cache: Dict[str, Any] = {
     "ts": 0,
     "df_offers": pd.DataFrame(),
     "raw_count": 0,
 }
 
-# ––––– Static Data (New Feature) –––––
+# ––––– Static Data (Fallbacks) –––––
 
 ZIM_ESSENTIALS = {
-    "fuel_petrol": "$1.58/L (Blend)",
-    "fuel_diesel": "$1.65/L (Diesel 50)",
-    "zesa_electricity": "Tiered: First 50 units cheap, then ~14c/kWh",
-    "bread_standard": "$1.00/loaf (Fixed)",
-    "gas_lpg": "$1.90 - $2.10 per kg"
+    "fuel": {"price": 1.58, "unit": "L", "retailer": "Pump Price"},
+    "petrol": {"price": 1.58, "unit": "L", "retailer": "Pump Price"},
+    "diesel": {"price": 1.65, "unit": "L", "retailer": "Pump Price"},
+    "bread": {"price": 1.00, "unit": "loaf", "retailer": "Standard"},
+    "gas": {"price": 2.00, "unit": "kg", "retailer": "LPG Market"},
+    "electricity": {"price": 20.00, "unit": "est. month", "retailer": "ZESA"},
+    "zesa": {"price": 20.00, "unit": "est. month", "retailer": "ZESA"},
 }
 
 # =========================
-# Helpers: time / strings
+# Helpers
 # =========================
 
 def now_utc_iso() -> str:
     return datetime.now(timezone.utc).isoformat()
 
-def _coerce_float(v: Any) -> Optional[float]:
+def _coerce_float(v: Any) -> float:
     try:
-        if v is None:
-            return None
-        if isinstance(v, (int, float)):
-            return float(v)
-        s = str(v).strip()
-        if not s:
-            return None
-        return float(s)
+        if v is None: return 0.0
+        return float(v)
     except Exception:
-        return None
+        return 0.0
 
 def _norm_str(s: Any) -> str:
     s = "" if s is None else str(s)
@@ -155,12 +139,17 @@ def _norm_str(s: Any) -> str:
 
 def _safe_json_loads(s: str, fallback: Any):
     try:
+        # Strip markdown code blocks if present
+        if "```json" in s:
+            s = s.split("```json")[1].split("```")[0]
+        elif "```" in s:
+            s = s.split("```")[0]
         return json.loads(s)
     except Exception:
         return fallback
 
 # =========================
-# Firestore profile storage
+# Firestore
 # =========================
 
 def profile_ref(profile_id: str):
@@ -168,14 +157,13 @@ def profile_ref(profile_id: str):
     return db.collection("pricelyst_profiles").document(profile_id)
 
 def get_profile(profile_id: str) -> Dict[str, Any]:
-    if not db:
-        return {}
+    if not db: return {}
     try:
         ref = profile_ref(profile_id)
         doc = ref.get()
         if doc.exists:
             return doc.to_dict() or {}
-        # create default
+        
         data = {
             "profile_id": profile_id,
             "created_at": now_utc_iso(),
@@ -183,956 +171,472 @@ def get_profile(profile_id: str) -> Dict[str, Any]:
             "username": None,
             "memory_summary": "",
             "preferences": {},
-            "last_actions": [],
-            "counters": {
-                "chats": 0,
-                "calls": 0,
-            }
+            "counters": {"chats": 0, "calls": 0}
         }
         ref.set(data)
         return data
     except Exception as e:
-        logger.error("get_profile error for %s: %s", profile_id, e)
+        logger.error("DB Error get_profile: %s", e)
         return {}
 
 def update_profile(profile_id: str, patch: Dict[str, Any]) -> None:
     if not db: return
     try:
-        patch = dict(patch or {})
+        patch = dict(patch)
         patch["updated_at"] = now_utc_iso()
         profile_ref(profile_id).set(patch, merge=True)
     except Exception as e:
-        logger.error("update_profile error: %s", e)
+        logger.error("DB Error update_profile: %s", e)
 
 def log_chat(profile_id: str, payload: Dict[str, Any]) -> None:
-    if not db:
-        logger.warning("DB not connected, skipping log_chat")
-        return
+    if not db: return
     try:
-        logger.info("Logging chat for %s. Type: %s", profile_id, payload.get("response_type"))
         db.collection("pricelyst_profiles").document(profile_id).collection("chat_logs").add({
             **payload,
             "ts": now_utc_iso()
         })
     except Exception as e:
-        logger.error("Failed to log chat: %s", e)
+        logger.error("DB Error log_chat: %s", e)
 
 def log_call(profile_id: str, payload: Dict[str, Any]) -> str:
-    if not db:
-        logger.warning("DB not connected, skipping log_call")
-        return ""
+    if not db: return str(int(time.time()))
     try:
-        logger.info("Logging call for %s. Transcript len: %s", profile_id, len(payload.get("transcript", "")))
-        doc_ref = db.collection("pricelyst_profiles").document(profile_id).collection("call_logs").document()
-        doc_ref.set({
+        ref = db.collection("pricelyst_profiles").document(profile_id).collection("call_logs").document()
+        ref.set({
             **payload,
             "ts": now_utc_iso()
         })
-        logger.info("Call logged successfully. ID: %s", doc_ref.id)
-        return doc_ref.id
+        return ref.id
     except Exception as e:
-        logger.error("Failed to log call: %s", e)
+        logger.error("DB Error log_call: %s", e)
         return ""
 
 # =========================
-# Multimodal image handling
+# Data Ingestion (ETL)
 # =========================
 
-def parse_images(images: List[str]) -> List[Dict[str, Any]]:
-    """
-    Accepts:
-    - data URLs: data:image/png;base64,....
-    - raw base64 strings
-    - http(s) URLs
-    Returns: list of { "mime": "...", "bytes": b"..." } or { "url": "..." }
-    """
-    out = []
-    for item in images or []:
-        if not item:
-            continue
-        item = item.strip()
-        
-        # URL
-        if item.startswith("http://") or item.startswith("https://"):
-            out.append({"url": item})
-            continue
-
-        # data URL
-        m = re.match(r"^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$", item)
-        if m:
-            mime = m.group(1)
-            b64 = m.group(2)
-            try:
-                out.append({"mime": mime, "bytes": base64.b64decode(b64)})
-            except Exception:
-                continue
-            continue
-        
-        # raw base64
+def fetch_products(max_pages: int = 10, per_page: int = 50) -> List[Dict[str, Any]]:
+    """Fetch raw products from Pricelyst API."""
+    all_products = []
+    for p in range(1, max_pages + 1):
         try:
-            out.append({"mime": "image/png", "bytes": base64.b64decode(item)})
-        except Exception:
-            continue
+            url = f"{PRICE_API_BASE}/api/v1/products"
+            r = requests.get(url, params={"page": p, "perPage": per_page}, timeout=HTTP_TIMEOUT)
+            r.raise_for_status()
+            data = r.json().get("data") or []
+            if not data: break
+            all_products.extend(data)
             
-    return out
-
-# =========================
-# Product fetching + offers DF
-# =========================
-
-def fetch_products_page(page: int, per_page: int = 50) -> Dict[str, Any]:
-    url = f"{PRICE_API_BASE}/api/v1/products"
-    params = {"page": page, "perPage": per_page}
-    r = requests.get(url, params=params, timeout=HTTP_TIMEOUT)
-    r.raise_for_status()
-    return r.json()
-
-def fetch_products(max_pages: int = 6, per_page: int = 50) -> List[Dict[str, Any]]:
-    products: List[Dict[str, Any]] = []
-    for p in range(1, max_pages + 1):
-        payload = fetch_products_page(p, per_page=per_page)
-        data = payload.get("data") or []
-        if isinstance(data, list):
-            products.extend(data)
-        total_pages = payload.get("totalPages")
-        if isinstance(total_pages, int) and p >= total_pages:
-            break
-        if not data:
+            # Pagination check
+            meta = r.json()
+            if p >= (meta.get("totalPages") or 999):
+                break
+        except Exception as e:
+            logger.warning(f"Product fetch error page {p}: {e}")
             break
-    return products
+    return all_products
 
-def products_to_offers_df(products: List[Dict[str, Any]]) -> pd.DataFrame:
+def flatten_products_to_df(products: List[Dict[str, Any]]) -> pd.DataFrame:
+    """
+    Strict mapping of the nested JSON structure to a flat search index.
+    Structure: product -> prices[] -> retailer
+    """
     rows = []
-    for p in products or []:
+    for p in products:
         try:
-            product_id = p.get("id")
-            name = p.get("name") or ""
-            clean_name = _norm_str(name)
-            
-            brand_name = ((p.get("brand") or {}).get("brand_name")) if isinstance(p.get("brand"), dict) else None
-            categories = p.get("categories") or []
-            cat_names = []
-            for c in categories:
-                if isinstance(c, dict) and c.get("name"):
-                    cat_names.append(c.get("name"))
-            primary_category = cat_names[0] if cat_names else None
+            p_id = p.get("id")
+            p_name = p.get("name") or "Unknown"
+            p_desc = p.get("description") or ""
+            p_slug = p.get("slug") or ""
             
-            stock_status = p.get("stock_status")
-            on_promo = bool(p.get("on_promotion"))
-            promo_badge = p.get("promo_badge")
-            promo_name = p.get("promo_name")
-            promo_price = _coerce_float(p.get("promo_price"))
-            original_price = _coerce_float(p.get("original_price"))
-            
-            recommended_price = _coerce_float(p.get("recommended_price"))
-            base_price = _coerce_float(p.get("price"))
-            bulk_price = _coerce_float(p.get("bulk_price"))
-            bulk_unit = p.get("bulk_unit")
-            
-            image = p.get("image")
-            thumb = p.get("thumbnail")
+            # Get Primary Category
+            cat_name = "General"
+            cats = p.get("categories") or []
+            if cats and isinstance(cats, list) and len(cats) > 0:
+                cat_name = cats[0].get("name") or "General"
+
+            # Brand
+            brand_name = (p.get("brand") or {}).get("brand_name") or ""
+
+            # Iterate Prices (Real Offers)
+            prices = p.get("prices") or []
             
-            offers = p.get("prices") or []
-            if not offers:
-                rows.append({
-                    "product_id": product_id,
-                    "product_name": name,
-                    "clean_name": clean_name,
-                    "brand_name": brand_name,
-                    "primary_category": primary_category,
-                    "categories": cat_names,
-                    "stock_status": stock_status,
-                    "on_promotion": on_promo,
-                    "promo_badge": promo_badge,
-                    "promo_name": promo_name,
-                    "promo_price": promo_price,
-                    "original_price": original_price,
-                    "recommended_price": recommended_price,
-                    "base_price": base_price,
-                    "bulk_price": bulk_price,
-                    "bulk_unit": bulk_unit,
-                    "image": image,
-                    "thumbnail": thumb,
-                    "retailer_id": None,
-                    "retailer_name": None,
-                    "retailer_type": None,
-                    "retailer_logo": None,
-                    "offer_price": None,
-                })
+            # If no prices, we still want the product indexed for knowledge, just no price
+            if not prices:
+                # Check for base price on product object as fallback
+                base_price = _coerce_float(p.get("price"))
+                if base_price > 0:
+                    rows.append({
+                        "product_id": p_id,
+                        "product_name": p_name,
+                        "clean_name": _norm_str(p_name),
+                        "description": p_desc,
+                        "category": cat_name,
+                        "brand": brand_name,
+                        "retailer": "Pricelyst Base", # Placeholder
+                        "price": base_price,
+                        "image": p.get("thumbnail") or p.get("image"),
+                    })
                 continue
 
-            for offer in offers:
-                if not isinstance(offer, dict):
-                    continue
-                retailer = offer.get("retailer") or {}
-                rows.append({
-                    "product_id": product_id,
-                    "product_name": name,
-                    "clean_name": clean_name,
-                    "brand_name": brand_name,
-                    "primary_category": primary_category,
-                    "categories": cat_names,
-                    "stock_status": stock_status,
-                    "on_promotion": on_promo,
-                    "promo_badge": promo_badge,
-                    "promo_name": promo_name,
-                    "promo_price": promo_price,
-                    "original_price": original_price,
-                    "recommended_price": recommended_price,
-                    "base_price": base_price,
-                    "bulk_price": bulk_price,
-                    "bulk_unit": bulk_unit,
-                    "image": image,
-                    "thumbnail": thumb,
-                    "retailer_id": offer.get("retailer_id") or retailer.get("id"),
-                    "retailer_name": (retailer.get("name") if isinstance(retailer, dict) else None),
-                    "retailer_type": (retailer.get("type") if isinstance(retailer, dict) else None),
-                    "retailer_logo": (retailer.get("logo") if isinstance(retailer, dict) else None),
-                    "offer_price": _coerce_float(offer.get("price")),
-                })
-        except Exception:
+            for offer in prices:
+                retailer_obj = offer.get("retailer") or {}
+                retailer_name = retailer_obj.get("name") or "Unknown Store"
+                price_val = _coerce_float(offer.get("price"))
+                
+                if price_val > 0:
+                    rows.append({
+                        "product_id": p_id,
+                        "product_name": p_name,
+                        "clean_name": _norm_str(p_name),
+                        "description": p_desc,
+                        "category": cat_name,
+                        "brand": brand_name,
+                        "retailer": retailer_name,
+                        "price": price_val,
+                        "image": p.get("thumbnail") or p.get("image"),
+                    })
+
+        except Exception as e:
             continue
 
     df = pd.DataFrame(rows)
-    if df.empty:
-        return df
-        
-    df["offer_price"] = df["offer_price"].apply(_coerce_float)
-    df["clean_name"] = df["clean_name"].fillna("").astype(str)
-    df["product_name"] = df["product_name"].fillna("").astype(str)
-    df["retailer_name"] = df["retailer_name"].fillna("").astype(str)
     return df
 
-def get_offers_df(force_refresh: bool = False) -> pd.DataFrame:
-    ts = _product_cache["ts"]
-    if (not force_refresh) and (time.time() - ts < PRODUCT_CACHE_TTL_SEC) and isinstance(_product_cache["df_offers"], pd.DataFrame) and not _product_cache["df_offers"].empty:
-        return _product_cache["df_offers"]
-
-    try:
-        products = fetch_products(max_pages=8, per_page=50)
-        df = products_to_offers_df(products)
-        _product_cache["ts"] = time.time()
-        _product_cache["df_offers"] = df
-        _product_cache["raw_count"] = len(products)
-        logger.info("Loaded offers DF: products=%s offers_rows=%s", len(products), len(df))
-        return df
-    except Exception as e:
-        logger.error("Failed to refresh product cache: %s", e)
-        # fallback: return old cache (even if stale)
-        if isinstance(_product_cache["df_offers"], pd.DataFrame):
-            return _product_cache["df_offers"]
-        return pd.DataFrame()
+def get_data_index(force_refresh: bool = False) -> pd.DataFrame:
+    """Singleton accessor for the product Dataframe."""
+    global _product_cache
+    
+    # Refresh if empty or stale
+    is_stale = (time.time() - _product_cache["ts"]) > PRODUCT_CACHE_TTL_SEC
+    if force_refresh or is_stale or _product_cache["df_offers"].empty:
+        logger.info("Refreshing Product Index...")
+        try:
+            raw_products = fetch_products(max_pages=15) # Fetch ~750 products
+            df = flatten_products_to_df(raw_products)
+            
+            _product_cache["ts"] = time.time()
+            _product_cache["df_offers"] = df
+            _product_cache["raw_count"] = len(raw_products)
+            logger.info(f"Index Refreshed: {len(df)} offers from {len(raw_products)} products.")
+        except Exception as e:
+            logger.error(f"Failed to refresh index: {e}")
+            
+    return _product_cache["df_offers"]
 
 # =========================
-# Gemini wrappers
+# Search & Matching Logic
 # =========================
 
-def gemini_generate_text(system: str, user: str, temperature: float = 0.4) -> str:
-    if not _gemini_client:
-        return ""
-    try:
-        resp = _gemini_client.models.generate_content(
-            model=GEMINI_MODEL,
-            contents=[
-                {"role": "user", "parts": [{"text": system.strip() + "\n\n" + user.strip()}]}
-            ],
-            config={
-                "temperature": temperature,
-                "max_output_tokens": 900,
-            }
-        )
-        return (resp.text or "").strip()
-    except Exception as e:
-        logger.error("Gemini text error: %s", e)
-        return ""
-
-def gemini_generate_json(system: str, user: str, images: List = None) -> Dict[str, Any]:
-    """NEW: Strict JSON generation for reliable Plan/Intent"""
-    if not _gemini_client: return {}
-    parts = [{"text": system + "\n\n" + user}]
-    for img in images or []:
-        if "bytes" in img:
-            b64 = base64.b64encode(img["bytes"]).decode("utf-8")
-            parts.append({"inline_data": {"mime_type": img["mime"], "data": b64}})
-        elif "url" in img:
-            parts.append({"text": f"Image URL: {img['url']}"})
-    try:
-        resp = _gemini_client.models.generate_content(
-            model=GEMINI_MODEL,
-            contents=[{"role": "user", "parts": parts}],
-            config={"temperature": 0.2, "response_mime_type": "application/json", "max_output_tokens": 2000}
-        )
-        return json.loads(resp.text)
-    except Exception as e:
-        logger.error("Gemini JSON error: %s", e)
-        return {}
-
-def gemini_generate_multimodal(system: str, user: str, images: List[Dict[str, Any]]) -> str:
+def search_index(df: pd.DataFrame, query: str, limit: int = 5) -> List[Dict[str, Any]]:
     """
-    Uses Gemini multimodal:
-    - if we have bytes -> inline_data
-    - if we have url -> just paste the URL (server-side fetch is unreliable w/o whitelisting),
-      so we prefer bytes from the client.
+    Search the DF using token overlap + substring matching.
+    Returns best distinct product matches with their best price.
     """
-    if not _gemini_client:
-        return ""
+    if df.empty: return []
+    
+    q_norm = _norm_str(query)
+    q_tokens = set(q_norm.split())
+    
+    # 1. Exact Substring Filter (Fast)
+    # matching_rows = df[df['clean_name'].str.contains(q_norm, regex=False)]
+    
+    # 2. Token Overlap Scoring (Better for "Cooking Oil" -> "Olivine Cooking Oil")
+    # We calculate a score 0-1 based on how many query tokens exist in product name
+    
+    def score_text(text):
+        if not isinstance(text, str): return 0
+        text_tokens = set(text.split())
+        if not text_tokens: return 0
+        intersection = q_tokens.intersection(text_tokens)
+        return len(intersection) / len(q_tokens) # % of query satisfied
+
+    # Copy to avoid warnings (this is in-memory, acceptable for <10k rows)
+    # For performance at scale, use vector DB or proper Search engine
+    temp_df = df.copy()
+    temp_df['score'] = temp_df['clean_name'].apply(score_text)
+    
+    # Filter for relevant matches (at least 50% token match or substring match)
+    matches = temp_df[ (temp_df['score'] > 0.4) | (temp_df['clean_name'].str.contains(q_norm, regex=False)) ]
+    
+    if matches.empty:
+        # Fallback: Try searching category
+        matches = temp_df[temp_df['category'].str.lower().str.contains(q_norm, na=False)]
 
-    parts: List[Dict[str, Any]] = [{"text": system.strip() + "\n\n" + user.strip()}]
+    if matches.empty:
+        return []
+
+    # Sort by Score desc, then Price asc
+    matches = matches.sort_values(by=['score', 'price'], ascending=[False, True])
+    
+    # Group by product_id to get unique products (Best Offer per product)
+    unique_products = []
+    seen_ids = set()
     
-    for img in images or []:
-        if "bytes" in img and img.get("mime"):
-            b64 = base64.b64encode(img["bytes"]).decode("utf-8")
-            parts.append({
-                "inline_data": {
-                    "mime_type": img["mime"],
-                    "data": b64
-                }
-            })
-        elif img.get("url"):
-            # last resort
-            parts.append({"text": f"[IMAGE_URL]\n{img['url']}"})
+    for _, row in matches.iterrows():
+        pid = row['product_id']
+        if pid in seen_ids: continue
+        seen_ids.add(pid)
+        
+        unique_products.append({
+            "id": pid,
+            "name": row['product_name'],
+            "price": row['price'],
+            "retailer": row['retailer'],
+            "category": row['category'],
+            "image": row['image']
+        })
+        if len(unique_products) >= limit: break
+        
+    return unique_products
 
+# =========================
+# Gemini Functions
+# =========================
+
+def gemini_generate_json(system_prompt: str, user_prompt: str) -> Dict[str, Any]:
+    if not _gemini_client: return {}
     try:
-        resp = _gemini_client.models.generate_content(
+        response = _gemini_client.models.generate_content(
             model=GEMINI_MODEL,
-            contents=[{"role": "user", "parts": parts}],
-            config={
-                "temperature": 0.2,
-                "max_output_tokens": 900,
-            }
+            contents=[
+                types.Content(role="user", parts=[
+                    types.Part.from_text(system_prompt + "\n\n" + user_prompt)
+                ])
+            ],
+            config=types.GenerateContentConfig(
+                response_mime_type="application/json",
+                temperature=0.2
+            )
         )
-        return (resp.text or "").strip()
+        return json.loads(response.text)
     except Exception as e:
-        logger.error("Gemini multimodal error: %s", e)
-        return ""
+        logger.error(f"Gemini JSON Error: {e}")
+        return {}
 
 # =========================
-# Intent + actionability
+# Shopping Plan Engine (RAG)
 # =========================
 
-INTENT_SYSTEM = """
-You are Pricelyst AI. Your job: understand whether the user is asking for actionable shopping help.
-Return STRICT JSON only.
-
-Output schema:
-{
-"actionable": true|false,
-"intent": one of [
-    "store_recommendation",
-    "price_lookup",
-    "price_compare",
-    "basket_optimize",
-    "basket_build",
-    "product_discovery",
-    "trust_check",
-    "chit_chat",
-    "lifestyle_lookup",
-    "other"
-],
-"items": [{"name": "...", "quantity": 1}],
-"constraints": {"budget": number|null, "location": "... "|null, "time_context": "mid-month|month-end|weekend|today|unknown"},
-"notes": "short reasoning"
-}
-
-Rules:
-- If user is chatting/social (hi, jokes, thanks, how are you, etc) => actionable=false, intent="chit_chat".
-- If user asks about prices/stores/basket/what to buy => actionable=true.
-- If user provided a list, extract items + quantities if obvious.
-- "How much is fuel/zesa/bread" -> lifestyle_lookup
-- Keep it conservative: if unclear, actionable=false.
+EXTRACT_SYSTEM_PROMPT = """
+You are a Shopping Assistant Data Extractor.
+Analyze the transcript and extract a list of shopping items the user implicitly or explicitly wants.
+Return JSON: { "items": [ { "name": "searchable term", "qty": "quantity string", "notes": "context" } ] }
+If no items found, return { "items": [] }.
 """
 
-def detect_intent(message: str, images_present: bool, context: Dict[str, Any]) -> Dict[str, Any]:
-    # 1. Fast path for ZIM_ESSENTIALS (Optimization)
-    msg_lower = message.lower()
-    for k in ZIM_ESSENTIALS:
-        clean_k = k.split('_')[-1] # fuel_petrol -> petrol
-        if clean_k in msg_lower and "price" in msg_lower:
-            return {"actionable": True, "intent": "lifestyle_lookup", "items": [{"name": k}]}
-
-    # 2. Gemini Detection
-    ctx_str = json.dumps(context or {}, ensure_ascii=False)
-    user = f"Message: {message}\nImagesPresent: {images_present}\nContext: {ctx_str}"
-
-    # Try using the strict JSON helper first for better reliability
-    try:
-        data = gemini_generate_json(INTENT_SYSTEM, user)
-        if not isinstance(data, dict): raise ValueError("Invalid JSON")
-    except:
-        # Fallback to text parsing if JSON mode fails (Backward Compat)
-        out = gemini_generate_text(INTENT_SYSTEM, user, temperature=0.1)
-        data = _safe_json_loads(out, fallback={})
-    
-    if not isinstance(data, dict):
-        return {"actionable": False, "intent": "other", "items": [], "constraints": {}, "notes": "bad_json"}
-    # normalize
-    data.setdefault("actionable", False)
-    data.setdefault("intent", "other")
-    data.setdefault("items", [])
-    data.setdefault("constraints", {})
-    return data
-
-# =========================
-# Shopping Plan Generator (NEW)
-# =========================
+SYNTHESIS_SYSTEM_PROMPT = """
+You are Jessica, Pricelyst's Shopping Advisor.
+Generate a shopping plan based on the USER TRANSCRIPT and the DATA CONTEXT provided.
 
-PLAN_SYSTEM_PROMPT = """
-You are Jessica, the Pricelyst Shopping Advisor. Analyze the conversation transcript.
-If the user discussed a shopping list, budget plan, or event needs, create a structured plan.
+RULES:
+1. USE REAL DATA: Use the prices and retailers found in DATA CONTEXT. Do not hallucinate prices.
+2. MISSING DATA: If an item is in the transcript but has "FOUND: False" in context, explicitly estimate it and mark it as "(Est)".
+3. FORMAT: Return strict JSON with a 'markdown_content' field containing a professional, formatted report (Tables, Totals, Tips).
 
-OUTPUT JSON SCHEMA:
+JSON SCHEMA:
 {
-"is_actionable": boolean,
-"title": "Short title (e.g. 'Weekend Braai List')",
-"summary": "1 sentence summary",
-"items": [{"name": "string", "qty": "string", "est_price": number|null}],
-"markdown_content": "A clean Markdown report for a PDF. Include headers (#), bullet points, and a budget summary table if applicable. Keep it professional."
+  "is_actionable": true,
+  "title": "Shopping Plan Title",
+  "markdown_content": "# Title\n\n..."
 }
-
-If no shopping/planning occurred, set is_actionable=false.
 """
 
-def generate_shopping_plan(transcript: str) -> Dict[str, Any]:
-    if not transcript or len(transcript) < 30:
-        return {"is_actionable": False}
-    return gemini_generate_json(PLAN_SYSTEM_PROMPT, f"TRANSCRIPT:\n{transcript}")
-
-# =========================
-# Matching + analytics
-# =========================
-
-def search_products(df: pd.DataFrame, query: str, limit: int = 10) -> pd.DataFrame:
+def build_shopping_plan(transcript: str) -> Dict[str, Any]:
     """
-    Simple search: contains on clean_name + fallback token overlap scoring.
+    RAG Pipeline:
+    1. Extract items from text.
+    2. Search DB for items.
+    3. Generate report using DB results.
     """
-    if df.empty:
-        return df
-
-    q = _norm_str(query)
-    if not q:
-        return df.head(0)
-    
-    # direct contains
-    hit = df[df["clean_name"].str.contains(re.escape(q), na=False)]
-    if len(hit) >= limit:
-        return hit.head(limit)
-    
-    # token overlap (cheap scoring)
-    q_tokens = set(q.split())
-    if not q_tokens:
-        return hit.head(limit)
-        
-    tmp = df.copy()
-    tmp["score"] = tmp["clean_name"].apply(lambda s: len(q_tokens.intersection(set(str(s).split()))))
-    tmp = tmp[tmp["score"] > 0].sort_values(["score"], ascending=False)
-    combined = pd.concat([hit, tmp], axis=0).drop_duplicates(subset=["product_id", "retailer_id"])
-    return combined.head(limit)
-
-def summarize_offers(df_hits: pd.DataFrame) -> Dict[str, Any]:
-    """
-    For one product name, there can be multiple retailers (offers).
-    We return:
-    - cheapest offer
-    - price range
-    - top offers
-    """
-    if df_hits.empty:
-        return {}
-    
-    # group by product_id (best is highest offer coverage)
-    grp = df_hits.groupby("product_id").size().sort_values(ascending=False)
-    best_pid = int(grp.index[0])
-    prod_rows = df_hits[df_hits["product_id"] == best_pid].copy()
-    
-    prod_name = prod_rows["product_name"].iloc[0]
-    brand = prod_rows["brand_name"].iloc[0]
-    category = prod_rows["primary_category"].iloc[0]
-    stock = prod_rows["stock_status"].iloc[0]
-    on_promo = bool(prod_rows["on_promotion"].iloc[0])
-    promo_badge = prod_rows["promo_badge"].iloc[0]
-    image = prod_rows["thumbnail"].iloc[0] or prod_rows["image"].iloc[0]
-    
-    offers = prod_rows[prod_rows["offer_price"].notna()].copy()
-    offers = offers.sort_values("offer_price", ascending=True)
-    
-    if offers.empty:
-        return {
-            "product_id": best_pid,
-            "name": prod_name,
-            "brand": brand,
-            "category": category,
-            "stock_status": stock,
-            "on_promotion": on_promo,
-            "promo_badge": promo_badge,
-            "image": image,
-            "offers": [],
-            "cheapest": None,
-            "price_range": None,
-        }
+    if len(transcript) < 10: 
+        return {"is_actionable": False}
 
-    cheapest = {
-        "retailer": offers.iloc[0]["retailer_name"],
-        "price": float(offers.iloc[0]["offer_price"] or 0),
-        "retailer_logo": offers.iloc[0]["retailer_logo"],
-    }
-    lo = float(offers["offer_price"].min())
-    hi = float(offers["offer_price"].max())
+    # Step 1: Extraction
+    extraction = gemini_generate_json(EXTRACT_SYSTEM_PROMPT, f"TRANSCRIPT:\n{transcript}")
+    items_requested = extraction.get("items", [])
     
-    top_offers = []
-    for _, r in offers.head(5).iterrows():
-        top_offers.append({
-            "retailer": r["retailer_name"],
-            "price": float(r["offer_price"]),
-            "retailer_logo": r["retailer_logo"],
-        })
-        
-    return {
-        "product_id": best_pid,
-        "name": prod_name,
-        "brand": brand,
-        "category": category,
-        "stock_status": stock,
-        "on_promotion": on_promo,
-        "promo_badge": promo_badge,
-        "image": image,
-        "offers": top_offers,
-        "cheapest": cheapest,
-        "price_range": {"min": lo, "max": hi, "spread": (hi - lo)},
-    }
+    if not items_requested:
+        return {"is_actionable": False}
 
-def basket_store_choice(df: pd.DataFrame, items: List[Dict[str, Any]]) -> Dict[str, Any]:
-    """
-    Given items, pick:
-    - best single store to cover most items and minimize total
-    Very pragmatic MVP: for each item, match the best product and take cheapest offer.
-    """
-    if df.empty or not items:
-        return {"items": [], "best_store": None, "missing": []}
+    df = get_data_index()
     
-    results = []
-    missing = []
+    # Step 2: Retrieval (The "Grounding")
+    context_lines = []
+    total_est = 0.0
     
-    for it in items:
-        name = it.get("name") or ""
-        qty = int(it.get("quantity") or 1)
-        hits = search_products(df, name, limit=50)
-        summary = summarize_offers(hits)
-        if not summary or not summary.get("cheapest"):
-            missing.append(name)
-            continue
-        cheapest = summary["cheapest"]
-        results.append({
-            "requested": name,
-            "matched_product": summary["name"],
-            "brand": summary.get("brand"),
-            "qty": qty,
-            "cheapest_retailer": cheapest["retailer"],
-            "unit_price": cheapest["price"],
-            "line_total": cheapest["price"] * qty,
-            "offers": summary.get("offers", []),
-            "image": summary.get("image"),
-        })
-        
-    if not results:
-        return {"items": [], "best_store": None, "missing": missing}
+    for item in items_requested:
+        term = item.get("name", "")
+        qty_str = item.get("qty", "1")
         
-    # compute totals by retailer for "all cheapest per item"
-    retailer_totals: Dict[str, float] = {}
-    retailer_counts: Dict[str, int] = {}
-    for r in results:
-        k = r["cheapest_retailer"]
-        retailer_totals[k] = retailer_totals.get(k, 0.0) + float(r["line_total"])
-        retailer_counts[k] = retailer_counts.get(k, 0) + 1
+        # Check ZIM_ESSENTIALS first
+        ess_key = next((k for k in ZIM_ESSENTIALS if k in term.lower()), None)
         
-    # Score: cover_count desc, then total asc
-    best = sorted(retailer_totals.keys(), key=lambda k: (-retailer_counts.get(k, 0), retailer_totals.get(k, 0.0)))[0]
-    return {
-        "items": results,
-        "best_store": {
-            "name": best,
-            "covered_items": retailer_counts.get(best, 0),
-            "total_for_covered_items": round(retailer_totals.get(best, 0.0), 2),
-            "total_items_requested": len(items),
-        },
-        "missing": missing
-    }
-
-# =========================
-# Response rendering (informative)
-# =========================
-
-def render_price_answer(summary: Dict[str, Any]) -> Dict[str, Any]:
-    """
-    Returns structured payload for frontend to render nicely.
-    """
-    if not summary:
-        return {
-            "type": "not_found",
-            "title": "I couldn't find that product.",
-            "message": "Try a different wording (brand + size helps), or upload an image/receipt.",
-        }
+        if ess_key:
+            data = ZIM_ESSENTIALS[ess_key]
+            price = data['price']
+            context_lines.append(f"- ITEM: {term} (Qty: {qty_str}) | FOUND: TRUE | SOURCE: Market Rate | PRICE: ${price} | RETAILER: {data['retailer']}")
+            total_est += price
+            continue
 
-    name = summary.get("name")
-    brand = summary.get("brand")
-    category = summary.get("category")
-    stock = summary.get("stock_status")
-    on_promo = summary.get("on_promotion")
-    promo_badge = summary.get("promo_badge")
-    image = summary.get("image")
-    cheapest = summary.get("cheapest")
-    pr = summary.get("price_range")
-    
-    lines = []
-    if cheapest:
-        lines.append(f"Cheapest right now: {cheapest['retailer']} — ${cheapest['price']:.2f}")
-    if pr and pr.get("min") is not None and pr.get("max") is not None and pr["max"] != pr["min"]:
-        lines.append(f"Price range: ${pr['min']:.2f} → ${pr['max']:.2f} (spread ${pr['spread']:.2f})")
-    if on_promo:
-        lines.append(f"Promo: {promo_badge or 'On promotion'}")
+        # Search Index
+        hits = search_index(df, term, limit=1)
         
-    return {
-        "type": "product_price",
-        "title": name,
-        "subtitle": " | ".join([x for x in [brand, category, stock] if x]),
-        "image": image,
-        "highlights": lines,
-        "offers": summary.get("offers", []),
-        "raw": summary,
-    }
-
-def render_basket_answer(basket: Dict[str, Any]) -> Dict[str, Any]:
-    if not basket.get("items"):
-        return {
-            "type": "basket_empty",
-            "title": "I couldn't build a basket from that.",
-            "message": "Send a clearer list (e.g., '2 bread, 1 cooking oil 2L') or upload a list/receipt photo."
-        }
-
-    best = basket.get("best_store")
-    missing = basket.get("missing") or []
-    return {
-        "type": "basket_plan",
-        "title": "Basket plan",
-        "best_store": best,
-        "items": basket["items"],
-        "missing": missing,
-        "notes": "If you want, tell me your budget and I'll suggest cheaper substitutes.",
-    }
-
-# =========================
-# Multimodal extraction (lists / receipts)
-# =========================
-
-VISION_SYSTEM = """
-You are an expert shopping assistant. Extract actionable items and quantities from the user's image(s).
-Return STRICT JSON only.
-
-Output schema:
-{
-"actionable": true|false,
-"items": [{"name":"...", "quantity": 1}],
-"notes": "short"
-}
+        if hits:
+            best = hits[0]
+            context_lines.append(f"- ITEM: {term} (Qty: {qty_str}) | FOUND: TRUE | PRODUCT: {best['name']} | PRICE: ${best['price']} | RETAILER: {best['retailer']}")
+            total_est += best['price']
+        else:
+            context_lines.append(f"- ITEM: {term} (Qty: {qty_str}) | FOUND: FALSE | NOTE: Needs estimation.")
 
-Rules:
-- If it looks like a handwritten shopping list, extract items.
-- If it looks like a receipt, extract the purchased items (best-effort).
-- If it's random (selfie, meme, etc), actionable=false and items=[].
-- Keep it conservative: only include items you're confident about.
-"""
+    data_context = "\n".join(context_lines)
+    logger.info(f"Plan Gen Context:\n{data_context}")
 
-def extract_items_from_images(images: List[Dict[str, Any]]) -> Dict[str, Any]:
-    if not images:
-        return {"actionable": False, "items": [], "notes": "no_images"}
-    user = "Extract items from the images."
-    out = gemini_generate_multimodal(VISION_SYSTEM, user, images)
-    data = _safe_json_loads(out, fallback={})
-    if not isinstance(data, dict):
-        return {"actionable": False, "items": [], "notes": "bad_json"}
-    data.setdefault("actionable", False)
-    data.setdefault("items", [])
-    return data
+    # Step 3: Synthesis
+    final_prompt = f"TRANSCRIPT:\n{transcript}\n\nDATA CONTEXT (Real Prices Found):\n{data_context}"
+    plan = gemini_generate_json(SYNTHESIS_SYSTEM_PROMPT, final_prompt)
+    
+    # Add metadata for frontend
+    plan["items_found"] = len([l for l in context_lines if "FOUND: TRUE" in l])
+    
+    return plan
 
 # =========================
-# Routes
+# API Endpoints
 # =========================
 
 @app.get("/health")
 def health():
+    df = get_data_index()
     return jsonify({
         "ok": True,
         "ts": now_utc_iso(),
-        "gemini": bool(_gemini_client),
-        "firestore": bool(db),
-        "products_cached_rows": int(len(_product_cache["df_offers"])) if isinstance(_product_cache["df_offers"], pd.DataFrame) else 0,
-        "products_raw_count": int(_product_cache.get("raw_count", 0)),
+        "db_connected": bool(db),
+        "products_indexed": len(df)
     })
 
 @app.post("/chat")
-def chat():
+def chat_endpoint():
+    """Text chat endpoint - kept mostly for legacy/debug, similar logic to voice."""
     body = request.get_json(silent=True) or {}
-    profile_id = (body.get("profile_id") or "").strip()
+    message = body.get("message", "")
+    profile_id = body.get("profile_id")
+    
     if not profile_id:
-        return jsonify({"ok": False, "error": "profile_id is required"}), 400
-
-    message = (body.get("message") or "").strip()
-    username = body.get("username")
-    context = body.get("context") or {}
-    images_raw = body.get("images") or []
-    images = parse_images(images_raw)
-
-    prof = get_profile(profile_id)
-    if username and not prof.get("username"):
-        update_profile(profile_id, {"username": username})
-
-    # 1) If images: try extract items (shopping list / receipt)
-    extracted = {"actionable": False, "items": [], "notes": "skipped"}
-    if images:
-        extracted = extract_items_from_images(images)
-
-    # 2) Detect intent from message (+ image presence)
-    intent = detect_intent(message, images_present=bool(images), context=context)
-
-    # If image extraction got items, treat as actionable unless the message is clearly chit-chat
-    image_items = extracted.get("items") if isinstance(extracted, dict) else []
-    if image_items and isinstance(image_items, list) and intent.get("intent") != "chit_chat":
-        intent["actionable"] = True
-        intent["intent"] = "basket_build" if len(image_items) > 1 else "price_lookup"
-        intent["items"] = image_items
-
-    # 3) Graceful conversational fallback
-    if not intent.get("actionable"):
-        reply = {
-            "type": "chat",
-            "message": (
-                f"Hey{(' ' + (username or prof.get('username') or '')).strip()} 👋\n"
-                "If you want shopping help, ask me something like:\n"
-                "• “Where is cooking oil cheapest?”\n"
-                "• “Which store is best for my basket: rice, chicken, oil?”\n"
-                "• “Build me a budget basket under $20.”"
-            )
-        }
-        # log + counters
-        log_chat(profile_id, {"message": message, "intent": intent, "response_type": "chit_chat"})
-        update_profile(profile_id, {"counters": {"chats": int((prof.get("counters") or {}).get("chats", 0)) + 1}})
-        return jsonify({"ok": True, "intent": intent, "data": reply})
+        return jsonify({"ok": False, "error": "No profile_id"}), 400
 
-    # 4) Actionable: execute
-    df = get_offers_df(force_refresh=False)
+    # Simple intent check for search
+    intent = "chat"
+    reply_data = {}
     
-    response_payload: Dict[str, Any] = {"type": "unknown", "message": "No result."}
-
-    # --- NEW: Check for Lifestyle/Essentials (Fuel/ZESA) ---
-    if intent["intent"] == "lifestyle_lookup":
-        # Items are auto-detected in detect_intent
-        key = intent["items"][0]["name"]
-        # Fuzzy match dict key
-        val = ZIM_ESSENTIALS.get(key) or ZIM_ESSENTIALS.get("fuel_petrol") # fallback
-        response_payload = {
-            "type": "info_card",
-            "title": f"Market Rate: {key.replace('_', ' ').title()}",
-            "message": str(val),
-            "highlights": [f"Current: {val}"]
-        }
-
-    # --- Original Logic ---
-    elif intent["intent"] in ("price_lookup", "trust_check", "product_discovery"):
-        # pick first item or treat message as query
-        query = ""
-        if intent.get("items"):
-            query = intent["items"][0].get("name") or ""
-        if not query:
-            query = message
-        hits = search_products(df, query, limit=80)
-        summary = summarize_offers(hits)
-        response_payload = render_price_answer(summary)
-
-    elif intent["intent"] in ("basket_build", "basket_optimize", "store_recommendation"):
-        items = intent.get("items") or []
-        # if user didn't provide items but asked store choice, we can try to extract nouns—too risky; keep conservative
-        if not items:
-            response_payload = {
-                "type": "need_list",
-                "title": "Send your list",
-                "message": "I can recommend the best store once you send your basket (even 3–5 items)."
+    if "price" in message.lower() or "find" in message.lower() or len(message.split()) < 5:
+        df = get_data_index()
+        hits = search_index(df, message, limit=3)
+        if hits:
+            intent = "product_found"
+            reply_data = {
+                "type": "product_card",
+                "products": hits
             }
         else:
-            basket = basket_store_choice(df, items)
-            response_payload = render_basket_answer(basket)
-
-    elif intent["intent"] == "price_compare":
-        items = intent.get("items") or []
-        if len(items) < 2:
-            response_payload = {
-                "type": "need_two_items",
-                "title": "Need two items",
-                "message": "Tell me two items to compare, e.g., “Coke 2L vs Pepsi 2L”."
-            }
-        else:
-            comparisons = []
-            for it in items[:3]:
-                hits = search_products(df, it.get("name") or "", limit=60)
-                summary = summarize_offers(hits)
-                comparisons.append(summary)
-            
-            # compute cheapest for each
-            rows = []
-            for s in comparisons:
-                if not s or not s.get("cheapest"):
-                    continue
-                rows.append({
-                    "name": s.get("name"),
-                    "cheapest_retailer": s["cheapest"]["retailer"],
-                    "price": s["cheapest"]["price"]
-                })
-            rows = sorted(rows, key=lambda x: x["price"])
-            response_payload = {
-                "type": "comparison",
-                "title": "Comparison",
-                "items": rows,
-                "winner": rows[0] if rows else None
-            }
+            reply_data = {"message": "I couldn't find that product in our database."}
+    else:
+        # Gemini Chat fallback
+        reply_data = {"message": "I can help you plan your shopping. Tell me what you need!"}
 
-    # 5) Persist + counters + light memory updates
-    log_chat(profile_id, {
-        "message": message,
-        "intent": intent,
-        "response_type": response_payload.get("type"),
-        "images_present": bool(images),
+    log_chat(profile_id, {"message": message, "intent": intent, "reply": reply_data})
+    
+    return jsonify({
+        "ok": True, 
+        "intent": {"actionable": True if intent == "product_found" else False}, 
+        "data": reply_data
     })
 
-    counters = prof.get("counters") or {}
-    update_profile(profile_id, {"counters": {"chats": int(counters.get("chats", 0)) + 1}})
-
-    # minimal preference inference
-    if response_payload.get("type") == "basket_plan" and response_payload.get("best_store"):
-        update_profile(profile_id, {"preferences": {"last_best_store": response_payload["best_store"]["name"]}})
-
-    return jsonify({"ok": True, "intent": intent, "data": response_payload})
-
 @app.post("/api/call-briefing")
 def call_briefing():
+    """
+    Called by Frontend before ElevenLabs starts.
+    Provides context (memory, user name, tone) to the AI Agent.
+    """
     body = request.get_json(silent=True) or {}
-    profile_id = (body.get("profile_id") or "").strip()
+    profile_id = body.get("profile_id")
+    username = body.get("username")
+    
     if not profile_id:
-        return jsonify({"ok": False, "error": "profile_id is required"}), 400
+        return jsonify({"ok": False, "error": "Missing profile_id"}), 400
 
-    username = body.get("username")
     prof = get_profile(profile_id)
-    
-    if username and not prof.get("username"):
+    if username:
         update_profile(profile_id, {"username": username})
-        prof["username"] = username
 
-    # Build lightweight "shopping intelligence" variables for ElevenLabs agent
-    prefs = prof.get("preferences") or {}
-    last_store = (prefs.get("last_best_store") or "").strip() or None
-    
-    # quick stats from recent chats (last 25)
-    intent_counts: Dict[str, int] = {}
-    try:
-        logs = db.collection("pricelyst_profiles").document(profile_id).collection("chat_logs") \
-            .order_by("ts", direction=firestore.Query.DESCENDING).limit(25).stream()
-        
-        intents = []
-        for d in logs:
-            dd = d.to_dict() or {}
-            ii = (dd.get("intent") or {}).get("intent")
-            if ii:
-                intents.append(ii)
-        
-        for ii in intents:
-            intent_counts[ii] = intent_counts.get(ii, 0) + 1
-    except Exception as e:
-        logger.error("Error fetching call briefing chat history: %s", e)
-
-    # --- KPI Snapshot Logic ---
-    # We construct a dictionary that the React client will pass as a JSON string
-    # We inject ZIM_ESSENTIALS here so the Agent has knowledge of fuel/zesa prices
+    # Shopping Intelligence Payload
     kpi_data = {
-        "username": prof.get("username") or "there",
-        "last_best_store": last_store,
-        "top_intents_last_25": sorted(intent_counts.items(), key=lambda x: x[1], reverse=True)[:5],
-        "tone": "practical_zimbabwe",
-        "market_rates_essentials": ZIM_ESSENTIALS  # <--- INJECTED KNOWLEDGE
+        "username": username or prof.get("username") or "there",
+        "market_rates": ZIM_ESSENTIALS,
+        "tone": "helpful_zimbabwean",
+        "system_note": "You are Jessica. Use the 'market_rates' for fuel/bread if asked. For other items, ask them what they need and say you will generate a plan after the call."
     }
 
     return jsonify({
         "ok": True,
-        "profile_id": profile_id,
         "memory_summary": prof.get("memory_summary", ""),
-        # This string is passed to ElevenLabs by the React Client
-        "kpi_snapshot": json.dumps(kpi_data) 
+        "kpi_snapshot": json.dumps(kpi_data)
     })
 
 @app.post("/api/log-call-usage")
 def log_call_usage():
+    """
+    Called by Frontend when call ends. 
+    1. Receives Transcript.
+    2. Generates RAG-based Shopping Plan.
+    3. Saves to Firestore.
+    """
     body = request.get_json(silent=True) or {}
-    profile_id = (body.get("profile_id") or "").strip()
-    if not profile_id:
-        return jsonify({"ok": False, "error": "profile_id is required"}), 400
-
-    transcript = (body.get("transcript") or "").strip()
-    call_id = body.get("call_id") or None
-    started_at = body.get("started_at") or None
-    ended_at = body.get("ended_at") or None
-    stats = body.get("stats") or {}
+    profile_id = body.get("profile_id")
+    transcript = body.get("transcript", "")
     
-    logger.info("Received call usage for %s. Transcript len: %d", profile_id, len(transcript))
+    if not profile_id: 
+        return jsonify({"ok": False, "error": "Missing profile_id"}), 400
 
-    prof = get_profile(profile_id)
+    logger.info(f"Processing Call for {profile_id}. Transcript Len: {len(transcript)}")
 
-    # --- UPGRADE: Use Shopping Plan Generator (JSON + Markdown) ---
-    plan_id = None
-    report_md = ""
+    # Generate Plan (Ground Truth)
     plan_data = {}
-
-    try:
-        if transcript:
-            logger.info("Generating shopping plan via Gemini...")
-            plan_data = generate_shopping_plan(transcript)
-            logger.info("Plan generated. actionable=%s, title=%s", plan_data.get("is_actionable"), plan_data.get("title"))
-
+    plan_id = None
+    
+    if len(transcript) > 20:
+        try:
+            plan_data = build_shopping_plan(transcript)
+            
             if plan_data.get("is_actionable"):
-                # Save structured plan
+                # Persist Plan
                 plan_ref = db.collection("pricelyst_profiles").document(profile_id).collection("shopping_plans").document()
                 plan_data["id"] = plan_ref.id
-                plan_data["call_id"] = call_id
                 plan_data["created_at"] = now_utc_iso()
                 plan_ref.set(plan_data)
                 plan_id = plan_ref.id
-                report_md = plan_data.get("markdown_content", "")
-                logger.info("Shopping plan stored. ID=%s", plan_id)
-            else:
-                logger.info("No actionable shopping plan found in call.")
-    except Exception as e:
-        logger.error("Error generating/storing shopping plan: %s", e)
-
-    # Log the call (link the plan_id)
-    doc_id = log_call(profile_id, {
-        "call_id": call_id,
-        "started_at": started_at,
-        "ended_at": ended_at,
-        "stats": stats,
+                logger.info(f"Plan Created: {plan_id}")
+        except Exception as e:
+            logger.error(f"Plan Gen Error: {e}")
+
+    # Log Call
+    log_call(profile_id, {
         "transcript": transcript,
-        "generated_plan_id": plan_id,
-        "report_markdown": report_md, 
+        "duration": body.get("duration_seconds"),
+        "plan_id": plan_id
     })
-    
-    # update counters
-    try:
-        counters = prof.get("counters") or {}
-        update_profile(profile_id, {"counters": {"calls": int(counters.get("calls", 0)) + 1}})
-    except Exception as e:
-        logger.error("Error updating profile counters: %s", e)
+
+    # Update Counters
+    prof = get_profile(profile_id)
+    cnt = prof.get("counters", {})
+    update_profile(profile_id, {"counters": {"calls": int(cnt.get("calls", 0)) + 1}})
 
     return jsonify({
         "ok": True,
-        "logged_call_doc_id": doc_id,
-        "shopping_plan": plan_data if plan_id else None # Frontend uses this for PDF
+        "shopping_plan": plan_data if plan_data.get("is_actionable") else None
     })
 
-# — NEW: Shopping Plans CRUD —
+# ––––– CRUD: Shopping Plans –––––
 
 @app.get("/api/shopping-plans")
 def list_plans():
@@ -1140,21 +644,9 @@ def list_plans():
     if not pid: return jsonify({"ok": False}), 400
     try:
         docs = db.collection("pricelyst_profiles").document(pid).collection("shopping_plans") \
-            .order_by("created_at", direction=firestore.Query.DESCENDING).limit(20).stream()
+                 .order_by("created_at", direction=firestore.Query.DESCENDING).limit(10).stream()
         plans = [{"id": d.id, **d.to_dict()} for d in docs]
         return jsonify({"ok": True, "plans": plans})
-    except Exception as e:
-        logger.error("list_plans error: %s", e)
-        return jsonify({"ok": False, "error": str(e)}), 500
-
-@app.get("/api/shopping-plans/<plan_id>")
-def get_plan(plan_id):
-    pid = request.args.get("profile_id")
-    if not pid: return jsonify({"ok": False}), 400
-    try:
-        doc = db.collection("pricelyst_profiles").document(pid).collection("shopping_plans").document(plan_id).get()
-        if not doc.exists: return jsonify({"ok": False, "error": "Not found"}), 404
-        return jsonify({"ok": True, "plan": doc.to_dict()})
     except Exception as e:
         return jsonify({"ok": False, "error": str(e)}), 500
 
@@ -1169,9 +661,14 @@ def delete_plan(plan_id):
         return jsonify({"ok": False, "error": str(e)}), 500
 
 # =========================
-# Run
+# Main
 # =========================
 
 if __name__ == "__main__":
-    port = int(os.environ.get("PORT", "7860"))
-    app.run(host="0.0.0.0", port=port, debug=True)
+    port = int(os.environ.get("PORT", 7860))
+    # Pre-warm cache on startup
+    try:
+        get_data_index(force_refresh=True)
+    except:
+        pass
+    app.run(host="0.0.0.0", port=port)