Update main.py

main.py

@@ -1,12 +1,12 @@
 """
-main.py — Pricelyst Shopping Advisor (Jessica Edition - Grounded Data Version)
+main.py — Pricelyst Shopping Advisor (Jessica Edition - Grounded Data & Memory)
 
 ✅ Flask API
 ✅ Firebase Admin persistence
-✅ Gemini via google-genai SDK
+✅ Gemini via google-genai SDK (Fixed & Robust)
 ✅ RAG (Retrieval Augmented Generation) for Shopping Plans
+✅ Long-Term Memory (Personal Details Extraction)
 ✅ Real Pricing Logic (No Hallucinations)
-✅ Backwards Compatible with React Client
 
 ENV VARS:
 - GOOGLE_API_KEY=...
@@ -70,7 +70,6 @@ def init_firestore_from_env() -> firestore.Client:
         return firestore.client()
 
     if not FIREBASE_ENV:
-        # Fallback for local dev if needed, or raise error
         logger.warning("FIREBASE env var missing. Persistence disabled.")
         return None
 
@@ -139,7 +138,7 @@ def _norm_str(s: Any) -> str:
 
 def _safe_json_loads(s: str, fallback: Any):
     try:
-        # Strip markdown code blocks if present
+        # Clean potential markdown wrapping
         if "```json" in s:
             s = s.split("```json")[1].split("```")[0]
         elif "```" in s:
@@ -188,16 +187,6 @@ def update_profile(profile_id: str, patch: Dict[str, Any]) -> None:
     except Exception as e:
         logger.error("DB Error update_profile: %s", e)
 
-def log_chat(profile_id: str, payload: Dict[str, Any]) -> None:
-    if not db: return
-    try:
-        db.collection("pricelyst_profiles").document(profile_id).collection("chat_logs").add({
-            **payload,
-            "ts": now_utc_iso()
-        })
-    except Exception as e:
-        logger.error("DB Error log_chat: %s", e)
-
 def log_call(profile_id: str, payload: Dict[str, Any]) -> str:
     if not db: return str(int(time.time()))
     try:
@@ -247,7 +236,6 @@ def flatten_products_to_df(products: List[Dict[str, Any]]) -> pd.DataFrame:
             p_id = p.get("id")
             p_name = p.get("name") or "Unknown"
             p_desc = p.get("description") or ""
-            p_slug = p.get("slug") or ""
             
             # Get Primary Category
             cat_name = "General"
@@ -261,9 +249,8 @@ def flatten_products_to_df(products: List[Dict[str, Any]]) -> pd.DataFrame:
             # Iterate Prices (Real Offers)
             prices = p.get("prices") or []
             
-            # If no prices, we still want the product indexed for knowledge, just no price
+            # Fallback if no prices found
             if not prices:
-                # Check for base price on product object as fallback
                 base_price = _coerce_float(p.get("price"))
                 if base_price > 0:
                     rows.append({
@@ -273,7 +260,7 @@ def flatten_products_to_df(products: List[Dict[str, Any]]) -> pd.DataFrame:
                         "description": p_desc,
                         "category": cat_name,
                         "brand": brand_name,
-                        "retailer": "Pricelyst Base", # Placeholder
+                        "retailer": "Pricelyst Base", 
                         "price": base_price,
                         "image": p.get("thumbnail") or p.get("image"),
                     })
@@ -307,12 +294,11 @@ def get_data_index(force_refresh: bool = False) -> pd.DataFrame:
     """Singleton accessor for the product Dataframe."""
     global _product_cache
     
-    # Refresh if empty or stale
     is_stale = (time.time() - _product_cache["ts"]) > PRODUCT_CACHE_TTL_SEC
     if force_refresh or is_stale or _product_cache["df_offers"].empty:
         logger.info("Refreshing Product Index...")
         try:
-            raw_products = fetch_products(max_pages=15) # Fetch ~750 products
+            raw_products = fetch_products(max_pages=15) 
             df = flatten_products_to_df(raw_products)
             
             _product_cache["ts"] = time.time()
@@ -321,6 +307,9 @@ def get_data_index(force_refresh: bool = False) -> pd.DataFrame:
             logger.info(f"Index Refreshed: {len(df)} offers from {len(raw_products)} products.")
         except Exception as e:
             logger.error(f"Failed to refresh index: {e}")
+            if isinstance(_product_cache["df_offers"], pd.DataFrame):
+                return _product_cache["df_offers"]
+            return pd.DataFrame()
             
     return _product_cache["df_offers"]
 
@@ -331,32 +320,23 @@ def get_data_index(force_refresh: bool = False) -> pd.DataFrame:
 def search_index(df: pd.DataFrame, query: str, limit: int = 5) -> List[Dict[str, Any]]:
     """
     Search the DF using token overlap + substring matching.
-    Returns best distinct product matches with their best price.
     """
     if df.empty: return []
     
     q_norm = _norm_str(query)
     q_tokens = set(q_norm.split())
     
-    # 1. Exact Substring Filter (Fast)
-    # matching_rows = df[df['clean_name'].str.contains(q_norm, regex=False)]
-    
-    # 2. Token Overlap Scoring (Better for "Cooking Oil" -> "Olivine Cooking Oil")
-    # We calculate a score 0-1 based on how many query tokens exist in product name
-    
     def score_text(text):
         if not isinstance(text, str): return 0
         text_tokens = set(text.split())
         if not text_tokens: return 0
         intersection = q_tokens.intersection(text_tokens)
-        return len(intersection) / len(q_tokens) # % of query satisfied
+        return len(intersection) / len(q_tokens)
 
-    # Copy to avoid warnings (this is in-memory, acceptable for <10k rows)
-    # For performance at scale, use vector DB or proper Search engine
     temp_df = df.copy()
     temp_df['score'] = temp_df['clean_name'].apply(score_text)
     
-    # Filter for relevant matches (at least 50% token match or substring match)
+    # Filter for relevant matches
     matches = temp_df[ (temp_df['score'] > 0.4) | (temp_df['clean_name'].str.contains(q_norm, regex=False)) ]
     
     if matches.empty:
@@ -369,7 +349,7 @@ def search_index(df: pd.DataFrame, query: str, limit: int = 5) -> List[Dict[str,
     # Sort by Score desc, then Price asc
     matches = matches.sort_values(by=['score', 'price'], ascending=[False, True])
     
-    # Group by product_id to get unique products (Best Offer per product)
+    # Unique product logic
     unique_products = []
     seen_ids = set()
     
@@ -391,29 +371,84 @@ def search_index(df: pd.DataFrame, query: str, limit: int = 5) -> List[Dict[str,
     return unique_products
 
 # =========================
-# Gemini Functions
+# Gemini Functions (FIXED & ROBUST)
 # =========================
 
+def gemini_generate_text(system_prompt: str, user_prompt: str) -> str:
+    """Standard text generation."""
+    if not _gemini_client: return ""
+    try:
+        # Simplified call using contents string directly
+        response = _gemini_client.models.generate_content(
+            model=GEMINI_MODEL,
+            contents=system_prompt + "\n\n" + user_prompt,
+            config=types.GenerateContentConfig(
+                temperature=0.4
+            )
+        )
+        return response.text or ""
+    except Exception as e:
+        logger.error(f"Gemini Text Error: {e}")
+        return ""
+
 def gemini_generate_json(system_prompt: str, user_prompt: str) -> Dict[str, Any]:
+    """JSON generation with strict parsing."""
     if not _gemini_client: return {}
     try:
         response = _gemini_client.models.generate_content(
             model=GEMINI_MODEL,
-            contents=[
-                types.Content(role="user", parts=[
-                    types.Part.from_text(system_prompt + "\n\n" + user_prompt)
-                ])
-            ],
+            contents=system_prompt + "\n\n" + user_prompt,
             config=types.GenerateContentConfig(
                 response_mime_type="application/json",
                 temperature=0.2
             )
         )
-        return json.loads(response.text)
+        return _safe_json_loads(response.text, {})
     except Exception as e:
         logger.error(f"Gemini JSON Error: {e}")
         return {}
 
+# =========================
+# Long Term Memory Engine
+# =========================
+
+MEMORY_SYSTEM_PROMPT = """
+You are the Memory Manager for Jessica, an AI Shopping Assistant.
+Your job is to update the User's "Memory Summary" based on their latest conversation.
+
+INPUTS:
+1. Current Memory: The existing summary of what we know about the user.
+2. New Transcript: The latest conversation.
+
+GOAL:
+Update the Current Memory to include new details. Focus on:
+- Names (User, Family, Friends)
+- Dietary preferences or allergies
+- Budget habits (e.g., "likes cheap meat", "buys bulk")
+- Life events (e.g., "hosting a braai on Friday", "wife's birthday")
+- Feedback (e.g., "loved the T-bone suggestion")
+
+OUTPUT:
+Return ONLY the updated text summary. Keep it concise (max 150 words).
+"""
+
+def update_long_term_memory(profile_id: str, transcript: str) -> None:
+    """Updates the user's profile memory summary based on the new call."""
+    if len(transcript) < 20: return
+
+    prof = get_profile(profile_id)
+    current_memory = prof.get("memory_summary", "")
+    
+    user_prompt = f"CURRENT MEMORY:\n{current_memory}\n\nNEW TRANSCRIPT:\n{transcript}"
+    
+    try:
+        new_memory = gemini_generate_text(MEMORY_SYSTEM_PROMPT, user_prompt)
+        if new_memory and len(new_memory) > 10:
+            update_profile(profile_id, {"memory_summary": new_memory})
+            logger.info(f"Memory updated for {profile_id}")
+    except Exception as e:
+        logger.error(f"Memory update failed: {e}")
+
 # =========================
 # Shopping Plan Engine (RAG)
 # =========================
@@ -421,7 +456,7 @@ def gemini_generate_json(system_prompt: str, user_prompt: str) -> Dict[str, Any]
 EXTRACT_SYSTEM_PROMPT = """
 You are a Shopping Assistant Data Extractor.
 Analyze the transcript and extract a list of shopping items the user implicitly or explicitly wants.
-Return JSON: { "items": [ { "name": "searchable term", "qty": "quantity string", "notes": "context" } ] }
+Return JSON: { "items": [ { "name": "searchable term", "qty": "quantity string" } ] }
 If no items found, return { "items": [] }.
 """
 
@@ -430,29 +465,26 @@ You are Jessica, Pricelyst's Shopping Advisor.
 Generate a shopping plan based on the USER TRANSCRIPT and the DATA CONTEXT provided.
 
 RULES:
-1. USE REAL DATA: Use the prices and retailers found in DATA CONTEXT. Do not hallucinate prices.
-2. MISSING DATA: If an item is in the transcript but has "FOUND: False" in context, explicitly estimate it and mark it as "(Est)".
-3. FORMAT: Return strict JSON with a 'markdown_content' field containing a professional, formatted report (Tables, Totals, Tips).
+1. USE REAL DATA: Use the prices and retailers found in DATA CONTEXT.
+2. ESTIMATES: If context says "FOUND: FALSE", use your best guess for Zimbabwe prices and mark as "(Est)".
+3. FORMAT: Return strict JSON with a 'markdown_content' field containing a professional report.
 
 JSON SCHEMA:
 {
   "is_actionable": true,
-  "title": "Shopping Plan Title",
+  "title": "Short Title",
   "markdown_content": "# Title\n\n..."
 }
 """
 
 def build_shopping_plan(transcript: str) -> Dict[str, Any]:
     """
-    RAG Pipeline:
-    1. Extract items from text.
-    2. Search DB for items.
-    3. Generate report using DB results.
+    RAG Pipeline: Extract -> Search -> Synthesize
     """
     if len(transcript) < 10: 
         return {"is_actionable": False}
 
-    # Step 1: Extraction
+    # 1. Extract
     extraction = gemini_generate_json(EXTRACT_SYSTEM_PROMPT, f"TRANSCRIPT:\n{transcript}")
     items_requested = extraction.get("items", [])
     
@@ -461,44 +493,35 @@ def build_shopping_plan(transcript: str) -> Dict[str, Any]:
 
     df = get_data_index()
     
-    # Step 2: Retrieval (The "Grounding")
+    # 2. Retrieval (Grounding)
     context_lines = []
-    total_est = 0.0
     
     for item in items_requested:
         term = item.get("name", "")
         qty_str = item.get("qty", "1")
         
-        # Check ZIM_ESSENTIALS first
+        # Check Essentials Fallback
         ess_key = next((k for k in ZIM_ESSENTIALS if k in term.lower()), None)
-        
         if ess_key:
             data = ZIM_ESSENTIALS[ess_key]
-            price = data['price']
-            context_lines.append(f"- ITEM: {term} (Qty: {qty_str}) | FOUND: TRUE | SOURCE: Market Rate | PRICE: ${price} | RETAILER: {data['retailer']}")
-            total_est += price
+            context_lines.append(f"- ITEM: {term} | SOURCE: Market Rate | PRICE: ${data['price']} | RETAILER: {data['retailer']}")
             continue
 
-        # Search Index
+        # Search DB
         hits = search_index(df, term, limit=1)
-        
         if hits:
             best = hits[0]
-            context_lines.append(f"- ITEM: {term} (Qty: {qty_str}) | FOUND: TRUE | PRODUCT: {best['name']} | PRICE: ${best['price']} | RETAILER: {best['retailer']}")
-            total_est += best['price']
+            context_lines.append(f"- ITEM: {term} | FOUND: TRUE | PRODUCT: {best['name']} | PRICE: ${best['price']} | RETAILER: {best['retailer']}")
         else:
-            context_lines.append(f"- ITEM: {term} (Qty: {qty_str}) | FOUND: FALSE | NOTE: Needs estimation.")
+            context_lines.append(f"- ITEM: {term} | FOUND: FALSE | NOTE: Needs estimation.")
 
     data_context = "\n".join(context_lines)
-    logger.info(f"Plan Gen Context:\n{data_context}")
+    logger.info(f"Plan Context:\n{data_context}")
 
-    # Step 3: Synthesis
-    final_prompt = f"TRANSCRIPT:\n{transcript}\n\nDATA CONTEXT (Real Prices Found):\n{data_context}"
+    # 3. Synthesis
+    final_prompt = f"TRANSCRIPT:\n{transcript}\n\nDATA CONTEXT (Real Prices):\n{data_context}"
     plan = gemini_generate_json(SYNTHESIS_SYSTEM_PROMPT, final_prompt)
     
-    # Add metadata for frontend
-    plan["items_found"] = len([l for l in context_lines if "FOUND: TRUE" in l])
-    
     return plan
 
 # =========================
@@ -511,52 +534,13 @@ def health():
     return jsonify({
         "ok": True,
         "ts": now_utc_iso(),
-        "db_connected": bool(db),
         "products_indexed": len(df)
     })
 
-@app.post("/chat")
-def chat_endpoint():
-    """Text chat endpoint - kept mostly for legacy/debug, similar logic to voice."""
-    body = request.get_json(silent=True) or {}
-    message = body.get("message", "")
-    profile_id = body.get("profile_id")
-    
-    if not profile_id:
-        return jsonify({"ok": False, "error": "No profile_id"}), 400
-
-    # Simple intent check for search
-    intent = "chat"
-    reply_data = {}
-    
-    if "price" in message.lower() or "find" in message.lower() or len(message.split()) < 5:
-        df = get_data_index()
-        hits = search_index(df, message, limit=3)
-        if hits:
-            intent = "product_found"
-            reply_data = {
-                "type": "product_card",
-                "products": hits
-            }
-        else:
-            reply_data = {"message": "I couldn't find that product in our database."}
-    else:
-        # Gemini Chat fallback
-        reply_data = {"message": "I can help you plan your shopping. Tell me what you need!"}
-
-    log_chat(profile_id, {"message": message, "intent": intent, "reply": reply_data})
-    
-    return jsonify({
-        "ok": True, 
-        "intent": {"actionable": True if intent == "product_found" else False}, 
-        "data": reply_data
-    })
-
 @app.post("/api/call-briefing")
 def call_briefing():
     """
-    Called by Frontend before ElevenLabs starts.
-    Provides context (memory, user name, tone) to the AI Agent.
+    Returns memory context to the frontend to pass to ElevenLabs.
     """
     body = request.get_json(silent=True) or {}
     profile_id = body.get("profile_id")
@@ -569,12 +553,12 @@ def call_briefing():
     if username:
         update_profile(profile_id, {"username": username})
 
-    # Shopping Intelligence Payload
+    # Prepare intelligence payload
     kpi_data = {
-        "username": username or prof.get("username") or "there",
+        "username": username or prof.get("username") or "Friend",
         "market_rates": ZIM_ESSENTIALS,
-        "tone": "helpful_zimbabwean",
-        "system_note": "You are Jessica. Use the 'market_rates' for fuel/bread if asked. For other items, ask them what they need and say you will generate a plan after the call."
+        "tone": "practical_zimbabwe",
+        "system_instruction": "You are Jessica. If user asks about 'how was the party?', check 'memory_summary' variable."
     }
 
     return jsonify({
@@ -586,10 +570,9 @@ def call_briefing():
 @app.post("/api/log-call-usage")
 def log_call_usage():
     """
-    Called by Frontend when call ends. 
-    1. Receives Transcript.
-    2. Generates RAG-based Shopping Plan.
-    3. Saves to Firestore.
+    1. Update Memory (Async logic, effectively)
+    2. Generate Shopping Plan (Ground Truth)
+    3. Persist Log
     """
     body = request.get_json(silent=True) or {}
     profile_id = body.get("profile_id")
@@ -600,7 +583,10 @@ def log_call_usage():
 
     logger.info(f"Processing Call for {profile_id}. Transcript Len: {len(transcript)}")
 
-    # Generate Plan (Ground Truth)
+    # 1. Update Long Term Memory
+    update_long_term_memory(profile_id, transcript)
+
+    # 2. Generate Plan
     plan_data = {}
     plan_id = None
     
@@ -609,7 +595,6 @@ def log_call_usage():
             plan_data = build_shopping_plan(transcript)
             
             if plan_data.get("is_actionable"):
-                # Persist Plan
                 plan_ref = db.collection("pricelyst_profiles").document(profile_id).collection("shopping_plans").document()
                 plan_data["id"] = plan_ref.id
                 plan_data["created_at"] = now_utc_iso()
@@ -619,18 +604,13 @@ def log_call_usage():
         except Exception as e:
             logger.error(f"Plan Gen Error: {e}")
 
-    # Log Call
+    # 3. Log Call
     log_call(profile_id, {
         "transcript": transcript,
         "duration": body.get("duration_seconds"),
         "plan_id": plan_id
     })
 
-    # Update Counters
-    prof = get_profile(profile_id)
-    cnt = prof.get("counters", {})
-    update_profile(profile_id, {"counters": {"calls": int(cnt.get("calls", 0)) + 1}})
-
     return jsonify({
         "ok": True,
         "shopping_plan": plan_data if plan_data.get("is_actionable") else None
@@ -666,7 +646,7 @@ def delete_plan(plan_id):
 
 if __name__ == "__main__":
     port = int(os.environ.get("PORT", 7860))
-    # Pre-warm cache on startup
+    # Pre-warm cache
     try:
         get_data_index(force_refresh=True)
     except: