Update main.py

main.py

@@ -1,19 +1,18 @@
 """
-main.py — Pricelyst Shopping Advisor (Analyst Edition - Full Context)
+main.py — Pricelyst Shopping Advisor (Jessica 2026 Edition)
 
 ✅ Flask API
 ✅ Firebase Admin Persistence
-✅ Gemini via google-genai SDK (Robust)
-✅ "Analyst Engine": Python Math for Baskets, ZESA, & Fuel
-✅ Ground Truth Data: Uses /api/v1/product-listing
-✅ Jessica Context: Injects Top 60 Real Products into Voice Agent
-✅ Intent Detection: Strict Casual vs Actionable separation
+✅ Gemini 2.5 Flash (Target Model)
+✅ "Analyst Engine": Python Math for Baskets & ZESA
+✅ "Creative Engine": 4-Step Hybrid Plans (Catalogue -> Estimates -> Totals -> Context Ideas)
+✅ Jessica Context: Live Mini-Catalogue Injection
 
 ENV VARS:
 - GOOGLE_API_KEY=...
 - FIREBASE='{"type":"service_account", ...}'
 - PRICE_API_BASE=https://api.pricelyst.co.zw
-- GEMINI_MODEL=gemini-2.0-flash
+- GEMINI_MODEL=gemini-2.5-flash
 - PORT=5000
 """
 
@@ -49,7 +48,8 @@ except Exception as e:
     logger.error("google-genai not installed. pip install google-genai. Error=%s", e)
 
 GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "")
-GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.0-flash")
+# Updated to 2.5 Flash as requested for 2026 timeline
+GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.5-flash")
 
 _gemini_client = None
 if genai and GOOGLE_API_KEY:
@@ -165,7 +165,6 @@ def fetch_and_flatten_data() -> pd.DataFrame:
             logger.error(f"ETL Error on page {page}: {e}")
             break
 
-    # Flattening Logic
     rows = []
     for p in all_products:
         try:
@@ -240,14 +239,13 @@ def get_market_index(force_refresh: bool = False) -> pd.DataFrame:
 
 def search_products_fuzzy(df: pd.DataFrame, query: str, limit: int = 10) -> pd.DataFrame:
     if df.empty or not query: return df
-    
     q_norm = _norm(query)
     
-    # 1. Broad Filter (Contains)
+    # 1. Contains
     mask_name = df['clean_name'].str.contains(q_norm, regex=False)
     matches = df[mask_name].copy()
     
-    # 2. If no exact contains, try token overlap
+    # 2. Token overlap fallback
     if matches.empty:
         q_tokens = set(q_norm.split())
         def token_score(text):
@@ -261,10 +259,9 @@ def search_products_fuzzy(df: pd.DataFrame, query: str, limit: int = 10) -> pd.D
         df_scored['score'] = df_scored['clean_name'].apply(token_score)
         matches = df_scored[df_scored['score'] > 0]
 
-    if matches.empty:
-        return matches
+    if matches.empty: return matches
 
-    # 3. Rank by Popularity (Views) + Price
+    # 3. Sort by Views (Popularity) + Price
     matches = matches.sort_values(by=['views', 'price'], ascending=[False, True])
     return matches.head(limit)
 
@@ -277,7 +274,7 @@ def calculate_basket_optimization(item_names: List[str]) -> Dict[str, Any]:
         logger.warning("Basket Engine: DF is empty.")
         return {"actionable": False, "error": "No data"}
 
-    logger.info(f"Basket Engine: Optimizing for {len(item_names)} items: {item_names}")
+    logger.info(f"Basket Engine: Optimizing for: {item_names}")
 
     found_items = [] 
     missing_global = []
@@ -289,7 +286,7 @@ def calculate_basket_optimization(item_names: List[str]) -> Dict[str, Any]:
             missing_global.append(item)
             continue
         
-        # Pick best match (First one is sorted by Views/Price)
+        # Best match based on popularity
         best_prod = hits.iloc[0]
         found_items.append({
             "query": item,
@@ -297,11 +294,16 @@ def calculate_basket_optimization(item_names: List[str]) -> Dict[str, Any]:
             "name": best_prod['product_name']
         })
 
+    # Note: We return actionable=True even if products missing, so Gemini can estimate
     if not found_items:
-        logger.info("Basket Engine: No items matched in DB.")
-        return {"actionable": False, "missing": missing_global}
-
-    # 2. Calculate Totals Per Retailer
+        return {
+            "actionable": True,
+            "basket_items": [],
+            "global_missing": missing_global,
+            "best_store": None
+        }
+
+    # 2. Calculate Retailer Totals
     target_pids = [x['product_id'] for x in found_items]
     relevant_offers = df[df['product_id'].isin(target_pids) & df['is_offer']]
     
@@ -310,11 +312,9 @@ def calculate_basket_optimization(item_names: List[str]) -> Dict[str, Any]:
     
     for retailer in all_retailers:
         r_df = relevant_offers[relevant_offers['retailer'] == retailer]
-        
         found_count = len(r_df)
         total_price = r_df['price'].sum()
         
-        # Identify misses
         retailer_pids = r_df['product_id'].tolist()
         found_names = [x['name'] for x in found_items if x['product_id'] in retailer_pids]
         
@@ -326,25 +326,20 @@ def calculate_basket_optimization(item_names: List[str]) -> Dict[str, Any]:
             "found_items": found_names
         })
 
-    # 3. Sort: Coverage Desc, Price Asc
     retailer_stats.sort(key=lambda x: (-x['coverage_percent'], x['total_price']))
-    
-    if not retailer_stats:
-        return {"actionable": False}
-
-    best_option = retailer_stats[0]
-    logger.info(f"Basket Engine: Best Store = {best_option['retailer']} (${best_option['total_price']})")
+    best_option = retailer_stats[0] if retailer_stats else None
 
     return {
         "actionable": True,
         "basket_items": [x['name'] for x in found_items],
+        "found_items_details": found_items,
         "global_missing": missing_global,
         "best_store": best_option,
         "all_stores": retailer_stats[:3]
     }
 
 def calculate_zesa_units(amount_usd: float) -> Dict[str, Any]:
-    remaining = amount_usd / 1.06 # Remove 6% levy approx
+    remaining = amount_usd / 1.06 
     units = 0.0
     breakdown = []
     
@@ -367,7 +362,7 @@ def calculate_zesa_units(amount_usd: float) -> Dict[str, Any]:
             t3 = ZIM_UTILITIES["zesa_step_3"]
             bought = remaining / t3["rate"]
             units += bought
-            breakdown.append(f"Balance ${(remaining + cost_t1 + cost_t2):.2f} -> {bought:.1f}u @ ${t3['rate']}")
+            breakdown.append(f"Balance -> {bought:.1f}u @ ${t3['rate']}")
         else:
             bought = remaining / t2["rate"]
             units += bought
@@ -384,28 +379,26 @@ def calculate_zesa_units(amount_usd: float) -> Dict[str, Any]:
     }
 
 # =========================
-# 3. Gemini Helpers (Strict)
+# 3. Gemini Helpers (Detect + Creative)
 # =========================
 
 def gemini_detect_intent(transcript: str) -> Dict[str, Any]:
-    """
-    Classifies if the conversation needs an Analyst action.
-    """
+    """Strict Intent Classification."""
     if not _gemini_client: return {"actionable": False}
     
     PROMPT = """
-    Analyze this transcript. Return STRICT JSON.
-    Is the user asking for shopping help (prices, basket, store advice, ZESA/Fuel)?
+    Analyze transcript. Return STRICT JSON.
+    Is user asking for shopping/prices/basket/utility help? (Actionable)
+    Or just saying hello/thanks? (Casual)
     
-    Output Schema:
+    Schema:
     {
       "actionable": boolean,
       "intent": "SHOPPING_BASKET" | "UTILITY_CALC" | "PRODUCT_SEARCH" | "CASUAL_CHAT",
-      "items": ["item1", "item2"] (if applicable),
-      "utility_amount": number (if applicable for ZESA/Fuel)
+      "items": ["list", "of", "items"],
+      "utility_amount": number
     }
     """
-    
     try:
         resp = _gemini_client.models.generate_content(
             model=GEMINI_MODEL,
@@ -417,29 +410,51 @@ def gemini_detect_intent(transcript: str) -> Dict[str, Any]:
         logger.error(f"Intent Detect Error: {e}")
         return {"actionable": False, "intent": "CASUAL_CHAT"}
 
-def gemini_chat_response(transcript: str, analyst_data: Dict) -> str:
-    if not _gemini_client: return "System offline."
-    
+def gemini_generate_4step_plan(transcript: str, analyst_result: Dict) -> str:
+    """
+    The Creative Engine: Generates the 4-Section Plan.
+    """
+    if not _gemini_client: return "# Error\nAI Offline."
+
     PROMPT = f"""
-    You are Jessica, Pricelyst Analyst.
-    User asked: "{transcript}"
+    You are Jessica, Pricelyst's Shopping Advisor.
+    Generate a formatted Markdown Shopping Plan.
     
-    DATA (Use this strictly):
-    {json.dumps(analyst_data, indent=2)}
+    USER CONTEXT: "{transcript}"
     
-    If 'actionable' is true, summarize the Best Store and Total Cost.
-    If ZESA data is present, give the units estimate.
-    Keep it short, helpful, and Zimbabwean.
-    """
+    ANALYST DATA (REAL PRICES):
+    {json.dumps(analyst_result, indent=2)}
+
+    REQUIRED SECTIONS (Do not deviate):
     
+    1. **In Our Catalogue ✅**
+       - List items found in 'found_items_details'.
+       - Show product name, retailer, and exact price from data.
+       
+    2. **Not in Catalogue (Estimates) 😔**
+       - List items from 'global_missing' or items in the user list but not in 'found_items'.
+       - Provide a realistic USD estimate for Zimbabwe.
+       
+    3. **Totals 💰**
+       - Confirmed Total (Catalogue)
+       - Estimated Total (Missing Items)
+       - Grand Total Estimate
+       
+    4. **Ideas & Tips 💡**
+       - Based on the user's event (e.g. Braai, Dinner) or items, suggest 3 creative ideas or forgotten items.
+       - Be fun but practical.
+       
+    Tone: Helpful, Zimbabwean. Use Markdown tables for lists.
+    """
     try:
         resp = _gemini_client.models.generate_content(
             model=GEMINI_MODEL,
             contents=PROMPT
         )
         return resp.text
-    except:
-        return "I have the data but couldn't summarize it."
+    except Exception as e:
+        logger.error(f"Creative Gen Error: {e}")
+        return "# Error\nCould not generate plan."
 
 # =========================
 # 4. Endpoints
@@ -456,35 +471,39 @@ def health():
 
 @app.post("/chat")
 def chat():
-    """Text Chat Interface."""
+    """Text Chat - Uses simple summary."""
     body = request.get_json(silent=True) or {}
     msg = body.get("message", "")
     pid = body.get("profile_id")
     
     if not pid: return jsonify({"ok": False}), 400
     
-    # 1. Detect Intent
     intent_data = gemini_detect_intent(msg)
     analyst_data = {}
     
-    # 2. Run Analyst (if actionable)
     if intent_data.get("actionable"):
         if intent_data["intent"] == "SHOPPING_BASKET" and intent_data.get("items"):
             analyst_data = calculate_basket_optimization(intent_data["items"])
         elif intent_data["intent"] == "UTILITY_CALC":
             analyst_data = calculate_zesa_units(intent_data.get("utility_amount", 20))
         elif intent_data["intent"] == "PRODUCT_SEARCH" and intent_data.get("items"):
-            # Reuse basket logic for single item search to get best store
             analyst_data = calculate_basket_optimization(intent_data["items"])
 
-    # 3. Generate Reply
-    reply = gemini_chat_response(msg, analyst_data)
-    
-    # Log
+    # Quick chat summary
+    reply = "I'm processing that for you."
+    if analyst_data:
+        try:
+            resp = _gemini_client.models.generate_content(
+                model=GEMINI_MODEL,
+                contents=f"Summarize this shopping data for chat:\n{json.dumps(analyst_data)}"
+            )
+            reply = resp.text
+        except: pass
+
     if db:
         db.collection("pricelyst_profiles").document(pid).collection("chat_logs").add({
             "message": msg,
-            "response_text": reply,
+            "response": reply,
             "intent": intent_data,
             "ts": datetime.now(timezone.utc).isoformat()
         })
@@ -494,7 +513,7 @@ def chat():
 @app.post("/api/call-briefing")
 def call_briefing():
     """
-    Injects Memory + Top Products Catalogue for the Voice Agent.
+    Injects Memory + Top 60 Mini-Catalogue.
     """
     body = request.get_json(silent=True) or {}
     pid = body.get("profile_id")
@@ -510,26 +529,24 @@ def call_briefing():
             prof = doc.to_dict()
         else:
             ref.set({"created_at": datetime.now(timezone.utc).isoformat()})
-    
+            
     if username and username != prof.get("username"):
         if db: db.collection("pricelyst_profiles").document(pid).set({"username": username}, merge=True)
 
-    # --- Generate Mini-Catalogue (Top 60 popular items) ---
+    # --- Mini-Catalogue Generation ---
     df = get_market_index()
-    top_products_str = ""
+    catalogue_str = ""
     if not df.empty:
-        # Sort by views desc, take top 60 unique product names
-        top_offers = df[df['is_offer']].sort_values('views', ascending=False).drop_duplicates('product_name').head(60)
-        # Format: "Name ($AvgPrice)"
-        items_list = []
-        for _, r in top_offers.iterrows():
-            items_list.append(f"{r['product_name']} (~${r['price']:.2f})")
-        top_products_str = ", ".join(items_list)
-
-    # Payload for ElevenLabs (Data Variables Only)
+        # Top 60 Popular items with Price
+        top = df[df['is_offer']].sort_values('views', ascending=False).drop_duplicates('product_name').head(60)
+        lines = []
+        for _, r in top.iterrows():
+            lines.append(f"{r['product_name']} (${r['price']:.2f})")
+        catalogue_str = ", ".join(lines)
+
     kpi_snapshot = {
         "market_rates": ZIM_UTILITIES,
-        "popular_products_catalogue": top_products_str
+        "popular_products": catalogue_str
     }
     
     return jsonify({
@@ -541,10 +558,12 @@ def call_briefing():
 @app.post("/api/log-call-usage")
 def log_call_usage():
     """
-    Post-Call Processor.
-    1. Intent Check (Strict).
-    2. Analyst Optimization.
-    3. Plan Gen & Persistence.
+    The Post-Call Orchestrator.
+    1. Update Memory.
+    2. Detect Intent.
+    3. Run Analyst.
+    4. Generate 4-Step Creative Plan.
+    5. Save.
     """
     body = request.get_json(silent=True) or {}
     pid = body.get("profile_id")
@@ -554,68 +573,53 @@ def log_call_usage():
     
     logger.info(f"Log Call: Processing {pid}. Transcript Len: {len(transcript)}")
 
-    # 1. Update Memory (Async-like)
+    # 1. Update Memory
     if len(transcript) > 20 and db:
         try:
             curr_mem = db.collection("pricelyst_profiles").document(pid).get().to_dict().get("memory_summary", "")
-            mem_prompt = f"Update user memory (concise) with new details:\nOLD: {curr_mem}\nTRANSCRIPT: {transcript}"
+            mem_prompt = f"Update user memory with details from transcript:\nOLD: {curr_mem}\nTRANSCRIPT: {transcript}"
             mem_resp = _gemini_client.models.generate_content(model=GEMINI_MODEL, contents=mem_prompt)
             db.collection("pricelyst_profiles").document(pid).set({"memory_summary": mem_resp.text}, merge=True)
         except Exception as e:
-            logger.error(f"Memory Update Failed: {e}")
+            logger.error(f"Memory Update Error: {e}")
 
-    # 2. Intent Detection (The Gatekeeper)
+    # 2. Intent Detection
     intent_data = gemini_detect_intent(transcript)
-    logger.info(f"Log Call: Intent detected: {intent_data.get('intent')}")
+    logger.info(f"Intent: {intent_data.get('intent')}")
     
     plan_data = {}
     
     # 3. Actionable Logic
     if intent_data.get("actionable"):
         
-        # Handle Shopping List
+        # Run Analyst
         if intent_data.get("items"):
             analyst_result = calculate_basket_optimization(intent_data["items"])
             
+            # 4. Generate Creative Plan (4-Step)
+            # We generate if we have actionable basket results OR missing items
             if analyst_result.get("actionable"):
-                best = analyst_result["best_store"]
-                
-                # Markdown Generation
-                md = f"# Shopping Plan\n\n"
-                md += f"**Recommended Store:** {best['retailer']}\n"
-                md += f"**Estimated Total:** ${best['total_price']:.2f}\n\n"
-                
-                md += "## Your Basket\n\n"
-                md += "| Item | Found? |\n|---|---|\n"
-                for it in analyst_result["basket_items"]:
-                    status = "✅ In Stock" if it in best["found_items"] else "❌ Not Found"
-                    md += f"| {it} | {status} |\n"
-                
-                if analyst_result["global_missing"]:
-                    md += "\n### Missing Items (Estimate Required)\n"
-                    for m in analyst_result["global_missing"]:
-                        md += f"- {m}\n"
+                md_content = gemini_generate_4step_plan(transcript, analyst_result)
                 
                 plan_data = {
                     "is_actionable": True,
-                    "title": f"Plan: {best['retailer']} (${best['total_price']:.2f})",
-                    "markdown_content": md,
+                    "title": f"Shopping Plan ({datetime.now().strftime('%d %b')})",
+                    "markdown_content": md_content,
                     "items": intent_data["items"],
                     "created_at": datetime.now(timezone.utc).isoformat()
                 }
                 
-                # Persist Plan
                 if db:
                     doc_ref = db.collection("pricelyst_profiles").document(pid).collection("shopping_plans").document()
                     plan_data["id"] = doc_ref.id
                     doc_ref.set(plan_data)
-                    logger.info(f"Log Call: Plan Saved {doc_ref.id}")
+                    logger.info(f"Plan Saved: {doc_ref.id}")
 
-    # 4. Log Call
+    # 5. Log Call
     if db:
         db.collection("pricelyst_profiles").document(pid).collection("call_logs").add({
             "transcript": transcript,
-            "intent_data": intent_data,
+            "intent": intent_data,
             "plan_generated": bool(plan_data),
             "ts": datetime.now(timezone.utc).isoformat()
         })
@@ -625,7 +629,7 @@ def log_call_usage():
         "shopping_plan": plan_data if plan_data.get("is_actionable") else None
     })
 
-# ––––– CRUD: Shopping Plans –––––
+# ––––– CRUD –––––
 
 @app.get("/api/shopping-plans")
 def list_plans():
@@ -655,7 +659,6 @@ def delete_plan(plan_id):
 
 if __name__ == "__main__":
     port = int(os.environ.get("PORT", 7860))
-    # Pre-warm Cache
     try:
         get_market_index(force_refresh=True)
     except: