Update chat backend to include agentic features

.gitignore

@@ -58,4 +58,10 @@ env/
 venv/
 ENV/
 env.bak/
-venv.bak/
+venv.bak/
+
+docs_research
+tools
+Real Estate Intelligence Layer.md
+grant_privileges.sql
+agent_prompt.md

Frontend_API_Updates.md

@@ -0,0 +1,38 @@
+# Frontend API Updates: Agentic Chat Upgrades
+
+This document outlines the recent architectural upgrades to the Real Estate Intelligence Layer chat endpoint (`/api/v1/real-estate/chat`). These changes introduce multi-turn conversational memory, interactive clarification buttons, document generation, and markdown formatting. 
+
+## 1. Multi-Turn Conversational Memory (Stateful `session_id`)
+
+The backend now fully supports multi-turn conversations by retaining chat history in-memory using the `session_id`.
+
+**Frontend Implementation:**
+- Ensure that the frontend generates a unique string for `session_id` when a chat session starts.
+- **CRITICAL:** You must pass the *same* `session_id` on every subsequent request within that chat session. If the `session_id` changes or is omitted, the backend will treat the message as a brand-new, isolated conversation and lose context.
+
+## 2. Interactive Clarification Buttons (`suggested_actions`)
+
+When a user's query is ambiguous or when a tool requires more parameters (e.g., asking for "market averages" without specifying a city), the LLM will now generate a set of clickable options to clarify the request instead of forcing the user to type.
+
+**Frontend Implementation:**
+- The `ChatResponse` model now includes a new optional field: `suggested_actions: List[str]`.
+- Example Response:
+  ```json
+  {
+    "reply": "I need more specific details. Here are some options to clarify your request:",
+    "path_used": "PATH_A",
+    "tools_called": [],
+    "suggested_actions": ["Rental Rates", "Property Prices", "Other (please specify)"]
+  }
+  ```
+- If the `suggested_actions` array is not empty, the frontend should render these strings as clickable buttons below the assistant's reply.
+- When a user clicks a button, the frontend should send the button's text as the next `message` payload (using the same `session_id`).
+
+## 3. Document Exports & Markdown Rendering
+
+The LLM is now strictly instructed to output its replies using **Markdown** formatting (including tables, bold text, lists). Furthermore, it has access to a new `generate_data_export` tool which can generate CSV or Markdown reports on-the-fly and upload them to Appwrite Storage.
+
+**Frontend Implementation:**
+- The frontend chat UI must be capable of rendering standard Markdown into HTML.
+- When the user asks to "download this data" or "export a CSV", the backend will generate the file, upload it to Appwrite, and return a Markdown link in the `reply` text (e.g., `[Download your CSV Report here](https://fra.cloud.appwrite.io/v1/storage/...)`). 
+- The frontend's markdown parser should seamlessly render this as a standard clickable anchor tag (`<a>`) so the user can download the file.

config.py

@@ -0,0 +1,13 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+GROQ_API_KEY = os.environ.get("GROQ_API_KEY", "")
+SUPABASE_URL = os.environ.get("SUPABASE_URL", "")
+SUPABASE_KEY = os.environ.get("SUPABASE_KEY", "")
+
+APP_WRITE_PROJECT_ID = os.environ.get("APP_WRITE_PROJECT_ID", "")
+APP_WRITE_API_ENDPOINT = os.environ.get("APP_WRITE_API_ENDPOINT", "")
+APP_WRITE_API_KEY = os.environ.get("APP_WRITE_API_KEY", "")
+APP_WRITE_BUCKET_ID = os.environ.get("APP_WRITE_BUCKET_ID", "")

kb_embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d025e65f12c0809aa1266d93e87c9c8da61ef1bad866b25ba3ccc3af825f832e
+size 23168

main.py

@@ -19,11 +19,15 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-print("Initializing Embedding Model...")
-embed_model = SentenceTransformer("all-MiniLM-L6-v2")
-texts = [d["text"] for d in KB_DOCS]
-kb_embeddings = embed_model.encode(texts, normalize_embeddings=True)
-print("Embeddings loaded into memory.")
+
+from services.embedding_service import get_embedding_model
+import unified_ingest
+
+# Ensure local and remote KBs are seeded idempotently
+unified_ingest.ensure_ingested()
+
+embed_model = get_embedding_model()
+kb_embeddings = np.load("kb_embeddings.npy")
 
 groq_client = Groq(api_key=os.environ.get("GROQ_API_KEY", ""))
 
@@ -77,3 +81,7 @@ def chat(req: ChatRequest):
     except Exception:
         answer = ask_groq(req.question, context, model="llama-3.1-8b-instant")
     return {"answer": answer}
+
+# Import and mount the new Real Estate router
+from routes import real_estate_chat
+app.include_router(real_estate_chat.router)

project_tree.txt

@@ -1,123 +0,0 @@
-alembic.ini
-compose.yaml
-Dockerfile
-pytest.ini
-README.Docker.md
-README.md
-requirements.txt
-
-alembic/
-    env.py
-    README
-    script.py.mako
-    __pycache__/
-    versions/
-        293bf8cae101_add_conversation_history_table.py
-        2bae362756b9_version_9.py
-        3bdda54c971f_update_embedding_dimension_to_3072.py
-
-inst/
-    backend_dashboard_implementation_logic.md
-    dashboard_logic_blueprint.md
-    project_description.md
-
-onnx_model/
-    config.json
-    model.onnx
-    special_tokens_map.json
-    tokenizer_config.json
-    tokenizer.json
-
-onnx_model_optimized/
-    config.json
-    model_optimized.onnx
-    model.onnx
-    ort_config.json
-    special_tokens_map.json
-    tokenizer_config.json
-    tokenizer.json
-
-outputs/
-    naija_stageB_report_20251009_080416.txt
-    naija_stageB_report_20251009_085514.txt
-    stageB_scores_20251009_080416.csv
-    stageB_scores_20251009_085514.csv
-
-reports/
-    sentiment_benchmark_2025-10-10_06-58-54.txt
-    sentiment_benchmark_2025-10-10_07-25-30.txt
-    sentiment_benchmark_2025-10-10_07-35-40.txt
-    sentiment_benchmark_2025-10-10_07-52-57.txt
-    sentiment_benchmark_2025-11-06_23-30-34.txt
-    sentiment_results_2025-10-10_06-58-54.csv
-    sentiment_results_2025-10-10_07-25-30.csv
-    sentiment_results_2025-10-10_07-35-40.csv
-    sentiment_results_2025-10-10_07-52-57.csv
-    sentiment_results_2025-11-06_23-30-34.csv
-
-sample_data/
-    cleaned_messages Chat with John.json
-    cleaned_messages Chat with SUPER EAGLES.json
-    cleaned_messages.json
-    time_segments Chat with John.json
-    time_segments Chat with SUPER EAGLES.json
-    time_segments.json
-
-scripts/
-    convert_to_onnx_optimized.py
-    convert_to_onnx.py
-    sentiment_infer_test_onnx.py
-    __pycache__/
-
-src/
-    app - Shortcut.lnk
-    app/
-        __init__.py
-        main.py
-        config.py
-        logging_config.py
-        limiter.py
-        security.py
-        schemas.py
-        models.py
-        crud.py
-        db/
-            base.py
-            session.py
-        routers/
-            __init__.py
-            auth.py
-            chats.py
-            dashboard.py
-            rag.py
-            rag_streamed.py
-            sentiment_dashboard.py
-            sentiment_sse.py
-            uploads.py
-        services/
-            dashboard_service.py
-            embedding_service.py
-            embedding_service_summary_version.py
-            embedding_worker.py
-            retrieval_service.py
-            retrieval_service_summary_version.py
-            rag_service.py
-            rag_service_prev.py
-            router_service.py
-            sentiment_worker.py
-            sentiment_worker1.py
-            sentiment_dashboard_service.py
-            summary_service.py
-        utils/
-            types.py
-            segment_chat.py
-            raw_txt_parser.py
-            pre_process.py
-            message_analytics.py
-            extract_file_name.py
-
-data/
-
-live/
-
-# Note: `tests/` and `venv312/` were intentionally excluded as requested.

requirements.txt

@@ -3,4 +3,5 @@ uvicorn[standard]==0.32.0
 sentence-transformers==3.3.1
 numpy==1.26.4
 groq==0.13.0
-python-dotenv==1.0.1
+python-dotenv==1.0.1
+supabase==2.7.4

routes/real_estate_chat.py

@@ -0,0 +1,50 @@
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, Depends, Request
+from pydantic import BaseModel
+
+from services.groq_service import process_chat_message
+from services.rate_limiter import limiter
+
+router = APIRouter(prefix="/api/v1/real-estate", tags=["Real Estate Intelligence Layer"])
+
+class ChatRequest(BaseModel):
+    message: str
+    session_id: str
+    context: Optional[Dict[str, Any]] = {}
+
+class ChatResponse(BaseModel):
+    reply: str
+    path_used: str
+    tools_called: List[Dict[str, Any]]
+    suggested_actions: Optional[List[str]] = []
+
+@router.post("/chat", response_model=ChatResponse)
+async def handle_real_estate_chat(payload: ChatRequest, request: Request):
+    # Enforce token bucket rate limiting by session ID or client IP
+    client_key = payload.session_id or request.client.host
+    limiter.check_rate_limit(client_key)
+    
+    result = await process_chat_message(
+        user_query=payload.message,
+        session_id=payload.session_id,
+        session_context=payload.context or {}
+    )
+    
+    return ChatResponse(
+        reply=result["reply"],
+        path_used=result["path_used"],
+        tools_called=result["tools_called"],
+        suggested_actions=result.get("suggested_actions", [])
+    )
+
+@router.get("/chat/starters")
+async def get_starter_prompts():
+    return {
+        "starters": [
+            "What's today's biggest rate spike?",
+            "What does the 7-day average mean?",
+            "Which Miami properties are unavailable right now?",
+            "How often is listing data refreshed?"
+        ]
+    }

services/appwrite_service.py

@@ -0,0 +1,49 @@
+import uuid
+import tempfile
+import os
+from appwrite.client import Client
+from appwrite.services.storage import Storage
+from appwrite.input_file import InputFile
+from config import (
+    APP_WRITE_PROJECT_ID,
+    APP_WRITE_API_ENDPOINT,
+    APP_WRITE_API_KEY,
+    APP_WRITE_BUCKET_ID
+)
+
+client = Client()
+client.set_endpoint(APP_WRITE_API_ENDPOINT)
+client.set_project(APP_WRITE_PROJECT_ID)
+client.set_key(APP_WRITE_API_KEY)
+
+storage = Storage(client)
+
+async def upload_document_to_appwrite(content: str, format: str) -> str:
+    """
+    Uploads a generated document to Appwrite storage and returns a view/download URL.
+    """
+    if format not in ["csv", "md"]:
+        format = "md"
+        
+    file_id = str(uuid.uuid4())
+    filename = f"report_{file_id}.{format}"
+    
+    temp_path = os.path.join(tempfile.gettempdir(), filename)
+    with open(temp_path, "w", encoding="utf-8") as f:
+        f.write(content)
+        
+    try:
+        result = storage.create_file(
+            bucket_id=APP_WRITE_BUCKET_ID,
+            file_id=file_id,
+            file=InputFile.from_path(temp_path)
+        )
+        
+        url = f"{APP_WRITE_API_ENDPOINT}/storage/buckets/{APP_WRITE_BUCKET_ID}/files/{file_id}/view?project={APP_WRITE_PROJECT_ID}"
+        return url
+    except Exception as e:
+        print(f"Appwrite Upload Error: {e}")
+        return f"Error uploading file: {e}"
+    finally:
+        if os.path.exists(temp_path):
+            os.remove(temp_path)

services/embedding_service.py

@@ -0,0 +1,16 @@
+from sentence_transformers import SentenceTransformer
+
+_embedder = None
+
+def get_embedding_model() -> SentenceTransformer:
+    global _embedder
+    if _embedder is None:
+        try:
+            # Try to load from local cache first to prevent blocking network requests
+            print("Loading embedding model from local cache...")
+            _embedder = SentenceTransformer("all-MiniLM-L6-v2", local_files_only=True)
+        except Exception:
+            # Fall back to downloading if not cached
+            print("Local cache not found. Downloading embedding model...")
+            _embedder = SentenceTransformer("all-MiniLM-L6-v2")
+    return _embedder

services/groq_service.py

@@ -0,0 +1,172 @@
+import json
+import re
+from groq import Groq
+from services.tools import REAL_ESTATE_TOOLS
+from services.supabase_service import execute_tool_rpc, search_methodology_rag
+from config import GROQ_API_KEY
+
+session_history = {}
+
+groq_client = Groq(api_key=GROQ_API_KEY)
+
+ROUTER_PROMPT = """You are the classification router for the Joule Dynamics Real Estate Intelligence Layer.
+Analyze the user query and classify it into EXACTLY ONE of five classifications:
+
+1. "OUT_OF_SCOPE": Query asks about Leads, Lead-capture, Pricing Monitor data, general web crawling, or cross-system topics outside of the /real-estate page.
+2. "PATH_A": Query asks a live-data question (prices, spikes, availability, market averages, KPIs, specific listing rates).
+3. "PATH_B": Query asks a methodology/system design question (7-day average definition, 2-night check-in window, 4x daily scrape cadence, Vrbo status, World Cup strategy).
+4. "BOTH": Query requires BOTH explaining a methodology concept AND fetching live data metrics.
+5. "GREETING": User is saying hello, thanking the assistant, or making casual conversation without asking a specific question.
+
+Respond ONLY with valid JSON matching this schema:
+{"classification": "OUT_OF_SCOPE" | "PATH_A" | "PATH_B" | "BOTH" | "GREETING", "reason": "1-sentence justification"}
+"""
+
+SYNTHESIS_PROMPT = """You are the B2B Real Estate Intelligence Assistant for Joule Dynamics.
+You provide precise data analysis to real estate investors and property managers reviewing short-term rental market performance.
+
+OPERATIONAL RULES:
+1. NEVER FABRICATE DATA: Rely strictly on returned tool outputs or retrieved methodology chunks.
+2. ZERO GUESSING: If data or methodology is missing, state plainly: "I don't have that information in the current real estate scope."
+3. SCOPE BOUNDARY: If asked about Leads or Price Monitors, state that you are currently scoped exclusively to the Real Estate Rate Monitor.
+4. FORMAT: Always format your final output in valid Markdown. Use bolding, lists, and tables to make data highly readable.
+5. CLARIFICATION: If the user's request is ambiguous or if a tool is missing parameters, you can ask a clarifying question. To provide clickable options to the user, include a specific JSON block at the very end of your response exactly like this:
+```json
+{"clarification_options": ["Option A", "Option B"]}
+```
+"""
+
+async def process_chat_message(user_query: str, session_id: str, session_context: dict) -> dict:
+    if session_id not in session_history:
+        session_history[session_id] = []
+
+    # STEP 1: Routing Classification (llama-3.1-8b-instant)
+    router_messages = [{"role": "system", "content": ROUTER_PROMPT}]
+    # To save tokens on routing, only include the last 4 messages of history
+    router_messages.extend(session_history[session_id][-4:])
+    router_messages.append({"role": "user", "content": user_query})
+
+    router_res = groq_client.chat.completions.create(
+        model="llama-3.1-8b-instant",
+        messages=router_messages,
+        temperature=0.0,
+        response_format={"type": "json_object"}
+    )
+    
+    routing = json.loads(router_res.choices[0].message.content)
+    classification = routing.get("classification", "PATH_A")
+
+    # Guardrail: Immediate short-circuit if Out of Scope
+    if classification == "OUT_OF_SCOPE":
+        return {
+            "reply": "I apologize, but I am currently scoped exclusively to the Real Estate Rate Monitor page. I cannot assist with other topics like lead generation, pricing automation, or general knowledge outside of real estate data.",
+            "path_used": "OUT_OF_SCOPE",
+            "tools_called": [],
+            "suggested_actions": []
+        }
+
+    if classification == "GREETING":
+        return {
+            "reply": "Hello! I'm the Joule Dynamics Real Estate Intelligence Assistant. I can help you with rate spikes, market trends, and availability data. How can I assist you today?",
+            "path_used": "GREETING",
+            "tools_called": [],
+            "suggested_actions": []
+        }
+
+    tool_results = []
+    rag_chunks = []
+
+    # STEP 2: Execute Vector Search if Path B or Both
+    if classification in ["PATH_B", "BOTH"]:
+        rag_chunks = await search_methodology_rag(user_query)
+
+    # STEP 2: Execute Vector Search if Path B or Both
+    if classification in ["PATH_B", "BOTH"]:
+        rag_chunks = await search_methodology_rag(user_query)
+
+    messages = [
+        {"role": "system", "content": SYNTHESIS_PROMPT}
+    ]
+    messages.extend(session_history[session_id])
+
+    user_msg_content = f"User Context Filters: {json.dumps(session_context)}\nUser Query: {user_query}"
+    messages.append({"role": "user", "content": user_msg_content})
+    session_history[session_id].append({"role": "user", "content": user_msg_content})
+
+    if rag_chunks:
+        messages.append({
+            "role": "system", 
+            "content": "Retrieved Methodology Context:\n" + "\n---\n".join(rag_chunks)
+        })
+
+    # STEP 3: Initial Brain Completion (llama-3.3-70b-versatile)
+    brain_res = groq_client.chat.completions.create(
+        model="llama-3.3-70b-versatile",
+        messages=messages,
+        tools=REAL_ESTATE_TOOLS if classification in ["PATH_A", "BOTH"] else None,
+        tool_choice="auto" if classification in ["PATH_A", "BOTH"] else "none",
+        temperature=0.2,
+        max_tokens=600
+    )
+
+    response_message = brain_res.choices[0].message
+
+    # STEP 4: Process Tool Calls if Triggered
+    if response_message.tool_calls:
+        messages.append(response_message)
+        for tool_call in response_message.tool_calls:
+            func_name = tool_call.function.name
+            func_args = json.loads(tool_call.function.arguments)
+            
+            if func_name == "generate_data_export":
+                from services.appwrite_service import upload_document_to_appwrite
+                url = await upload_document_to_appwrite(func_args.get("content", ""), func_args.get("format", "md"))
+                db_result = {"status": "success", "url": url}
+            else:
+                db_result = await execute_tool_rpc(func_name, func_args)
+                
+            tool_results.append({"tool": func_name, "args": func_args})
+
+            messages.append({
+                "tool_call_id": tool_call.id,
+                "role": "tool",
+                "name": func_name,
+                "content": json.dumps(db_result)
+            })
+
+        # Second Brain Call to Synthesize Final Output
+        final_res = groq_client.chat.completions.create(
+            model="llama-3.3-70b-versatile",
+            messages=messages,
+            temperature=0.2,
+            max_tokens=600
+        )
+        final_reply = final_res.choices[0].message.content
+    else:
+        final_reply = response_message.content
+
+    # Track assistant reply in history
+    session_history[session_id].append({"role": "assistant", "content": final_reply})
+    if len(session_history[session_id]) > 20:
+        session_history[session_id] = session_history[session_id][-20:]
+
+    # Parse clarification options
+    suggested_actions = []
+    json_match = re.search(r'```json\s*(\{.*"clarification_options".*\})\s*```', final_reply, re.DOTALL)
+    if not json_match:
+        json_match = re.search(r'(\{.*"clarification_options".*\})', final_reply, re.DOTALL)
+        
+    if json_match:
+        try:
+            clarification_data = json.loads(json_match.group(1))
+            suggested_actions = clarification_data.get("clarification_options", [])
+            final_reply = final_reply.replace(json_match.group(0), "").strip()
+        except json.JSONDecodeError:
+            pass
+
+    return {
+        "reply": final_reply,
+        "path_used": classification,
+        "tools_called": tool_results,
+        "suggested_actions": suggested_actions
+    }

services/rate_limiter.py

@@ -0,0 +1,40 @@
+import time
+from collections import defaultdict
+from fastapi import HTTPException, Request
+
+class SessionRateLimiter:
+    def __init__(self, requests_per_minute: int = 10, max_daily: int = 50):
+        self.rpm = requests_per_minute
+        self.max_daily = max_daily
+        self.requests = defaultdict(list)
+        self.daily_counts = defaultdict(lambda: {"count": 0, "reset_at": 0})
+
+    def check_rate_limit(self, client_id: str):
+        now = time.time()
+        
+        # Check 24-hour Daily Limit
+        daily = self.daily_counts[client_id]
+        if now > daily["reset_at"]:
+            daily["count"] = 0
+            daily["reset_at"] = now + 86400
+            
+        if daily["count"] >= self.max_daily:
+            raise HTTPException(
+                status_code=429, 
+                detail="Daily limit of 50 messages reached for this session."
+            )
+
+        # Check RPM Window Limit
+        window = [t for t in self.requests[client_id] if now - t < 60]
+        self.requests[client_id] = window
+        
+        if len(window) >= self.rpm:
+            raise HTTPException(
+                status_code=429, 
+                detail="Rate limit exceeded. Please wait 1 minute before sending another question."
+            )
+            
+        self.requests[client_id].append(now)
+        daily["count"] += 1
+
+limiter = SessionRateLimiter()

services/supabase_service.py

@@ -0,0 +1,33 @@
+import os
+from supabase import create_client, Client
+from services.embedding_service import get_embedding_model
+from config import SUPABASE_URL, SUPABASE_KEY
+
+supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
+embedder = get_embedding_model()
+
+async def execute_tool_rpc(func_name: str, args: dict) -> dict:
+    """Executes a read-only Supabase RPC matching the tool schema."""
+    try:
+        res = supabase.rpc(func_name, args).execute()
+        return {"status": "success", "data": res.data}
+    except Exception as e:
+        return {"status": "error", "message": str(e)}
+
+async def search_methodology_rag(query: str, top_k: int = 3) -> list:
+    """Embeds query and retrieves top matching methodology documentation chunks."""
+    try:
+        vector = embedder.encode(query).tolist()
+        res = supabase.rpc(
+            "match_re_methodology",
+            {
+                "query_embedding": vector,
+                "match_threshold": 0.45,
+                "match_count": top_k
+            }
+        ).execute()
+        
+        return [f"### {item['section_title']}\n{item['chunk_content']}" for item in res.data]
+    except Exception as e:
+        print(f"RAG Retrieval Error: {e}")
+        return []

services/tools.py

@@ -0,0 +1,86 @@
+REAL_ESTATE_TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "get_dashboard_kpis",
+            "description": "Fetch overall Real Estate Rate Monitor KPIs including tracked properties count, 7D rate changes, 25%+ spikes, and scrape health.",
+            "parameters": {"type": "object", "properties": {}, "required": []}
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_market_averages",
+            "description": "Get average, minimum, and maximum nightly rates aggregated by regional market (e.g., 'NYC/NJ Metro' or 'Miami').",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "market": {"type": "string", "description": "Market region name, e.g. 'Miami' or 'NYC/NJ Metro'"}
+                },
+                "required": []
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_spike_alerts",
+            "description": "Fetch properties experiencing rate volatility spikes or drops equal to or exceeding a percentage deviation threshold from their 7-day average.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "threshold": {"type": "number", "default": 25.0, "description": "Minimum absolute percentage deviation from 7-day trailing average"},
+                    "days": {"type": "integer", "default": 7, "description": "Lookback window in days"}
+                },
+                "required": []
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_property_rate_history",
+            "description": "Retrieve historical nightly rate time series and 7-day trailing average benchmark for a specific property.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "property_search": {"type": "string", "description": "Property name or UUID"},
+                    "days": {"type": "integer", "default": 30}
+                },
+                "required": ["property_search"]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_properties_by_filter",
+            "description": "Search and filter the property rate snapshot table by market, platform, availability, or bedroom count.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "market": {"type": "string", "description": "e.g., 'Miami' or 'NYC/NJ Metro'"},
+                    "platform": {"type": "string", "description": "e.g., 'airbnb'"},
+                    "available": {"type": "boolean", "description": "true for available now, false for unavailable"},
+                    "bedrooms": {"type": "integer", "description": "Number of bedrooms"}
+                },
+                "required": []
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "generate_data_export",
+            "description": "Generate a downloadable report or data export (CSV or Markdown) based on data and metrics the user wants to keep.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "format": {"type": "string", "description": "Format of the export: 'csv' or 'md'"},
+                    "content": {"type": "string", "description": "The full text content or CSV data to be exported into the file"}
+                },
+                "required": ["format", "content"]
+            }
+        }
+    }
+]

test_api.py

@@ -0,0 +1,59 @@
+from fastapi.testclient import TestClient
+from main import app
+import json
+
+client = TestClient(app)
+
+print("--- Testing /health ---")
+try:
+    resp = client.get("/health")
+    print(f"Status: {resp.status_code}")
+    print(f"Response: {resp.json()}")
+except Exception as e:
+    print(f"Failed: {e}")
+
+print("\n--- Testing /api/v1/real-estate/chat/starters ---")
+try:
+    resp = client.get("/api/v1/real-estate/chat/starters")
+    print(f"Status: {resp.status_code}")
+    print(f"Response: {resp.json()}")
+except Exception as e:
+    print(f"Failed: {e}")
+
+messages = [
+    "Hello there!",
+    "What does the 7-day average mean?",
+    "Can you export the market averages for Miami to a CSV file?"
+]
+
+print("\n--- Testing /api/v1/real-estate/chat ---")
+
+# First, test isolated queries
+for i, msg in enumerate(messages):
+    print(f"\nTest {i+1}: '{msg}'")
+    try:
+        payload = {"message": msg, "session_id": f"isolated_session_{i}"}
+        resp = client.post("/api/v1/real-estate/chat", json=payload)
+        print(f"Status: {resp.status_code}")
+        print(f"Response: {json.dumps(resp.json(), indent=2)}")
+    except Exception as e:
+        print(f"Failed: {e}")
+
+# Next, test multi-turn conversation
+print("\n--- Testing Multi-Turn Conversation & Memory ---")
+multi_turn_msgs = [
+    "What's the market average for Miami?",
+    "Can you export that data into a CSV file for me?",
+    "Are there any other markets you track? I'm not sure which one I want."
+]
+session_id = "multi_turn_test_session_1"
+
+for i, msg in enumerate(multi_turn_msgs):
+    print(f"\nTurn {i+1}: '{msg}'")
+    try:
+        payload = {"message": msg, "session_id": session_id}
+        resp = client.post("/api/v1/real-estate/chat", json=payload)
+        print(f"Status: {resp.status_code}")
+        print(f"Response: {json.dumps(resp.json(), indent=2)}")
+    except Exception as e:
+        print(f"Failed: {e}")

unified_ingest.py

@@ -0,0 +1,87 @@
+import os
+import uuid
+import numpy as np
+from services.embedding_service import get_embedding_model
+from kb_docs import KB_DOCS
+from supabase import create_client, Client
+from config import SUPABASE_URL, SUPABASE_KEY
+
+METHODOLOGY_DOCS = [
+    {
+        "section_title": "Availability & 2-Night Window Definition",
+        "chunk_content": "A listing marked as 'Unavailable' means no open booking dates were detected within a 2-night check-in window starting from the current date. The system does not look ahead across full calendar months; it tracks consecutive 2-night availability as an immediate signal."
+    },
+    {
+        "section_title": "7-Day Trailing Average Benchmark",
+        "chunk_content": "The 7-day trailing average rate is calculated per property by taking the mean nightly price recorded across the prior 6 daily scrapes. Rate volatility alerts trigger when a property's current nightly rate strays 25% or more above or below this baseline."
+    },
+    {
+        "section_title": "Scrape Cadence & Data Refresh",
+        "chunk_content": "Listings are scraped 4 times daily to capture short-term rate adjustments and booking updates. Status 'Stale' indicates a scraper job failed to return fresh price points within the last 12 hours."
+    },
+    {
+        "section_title": "World Cup 2026 Strategic Focus",
+        "chunk_content": "The Real Estate Rate Monitor specifically tracks short-term rental inventory across NYC/NJ Metro and Miami markets to capture rate surges, supply constraints, and pricing dynamic anomalies leading up to the 2026 World Cup Final."
+    },
+    {
+        "section_title": "Vrbo Historical Tracking Status",
+        "chunk_content": "Vrbo properties are flagged as 'Historical' following platform scraping accessibility adjustments. Historical listings remain visible for baseline comparisons, but fresh daily rates are actively tracked via Airbnb endpoints."
+    }
+]
+
+def ensure_ingested():
+    """
+    Idempotent function that seeds both the local numpy embeddings and the remote Supabase database.
+    """
+    print("Verifying ingestion state...")
+    embedder = get_embedding_model()
+    
+    # 1. Local Numpy KB for Amara (Idempotent)
+    if not os.path.exists("kb_embeddings.npy"):
+        print("kb_embeddings.npy not found, generating local embeddings...")
+        texts = [d["text"] for d in KB_DOCS]
+        kb_embeddings = embedder.encode(texts, normalize_embeddings=True)
+        np.save("kb_embeddings.npy", kb_embeddings)
+        print(f"Embedded {len(texts)} KB docs and saved to kb_embeddings.npy")
+    else:
+        # Check shape to ensure it's valid, otherwise overwrite
+        try:
+            arr = np.load("kb_embeddings.npy")
+            if len(arr) != len(KB_DOCS):
+                raise ValueError("Mismatch length")
+        except Exception:
+            print("kb_embeddings.npy is corrupted or outdated. Regenerating...")
+            texts = [d["text"] for d in KB_DOCS]
+            kb_embeddings = embedder.encode(texts, normalize_embeddings=True)
+            np.save("kb_embeddings.npy", kb_embeddings)
+            print(f"Embedded {len(texts)} KB docs and saved to kb_embeddings.npy")
+            
+    # 2. Remote Supabase KB for Real Estate (Idempotent)
+    supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
+    
+    # Check what already exists in Supabase
+    try:
+        existing_res = supabase.table("re_knowledge_base").select("section_title").execute()
+        existing_titles = {row["section_title"] for row in existing_res.data}
+    except Exception as e:
+        print(f"Failed to query Supabase: {e}")
+        existing_titles = set()
+        
+    for doc in METHODOLOGY_DOCS:
+        if doc["section_title"] not in existing_titles:
+            print(f"Seeding missing chunk to Supabase: {doc['section_title']}")
+            vector = embedder.encode(doc["chunk_content"]).tolist()
+            payload = {
+                "id": str(uuid.uuid4()),
+                "section_title": doc["section_title"],
+                "chunk_content": doc["chunk_content"],
+                "embedding": vector
+            }
+            try:
+                supabase.table("re_knowledge_base").insert(payload).execute()
+                print(f"Successfully seeded: {doc['section_title']}")
+            except Exception as e:
+                print(f"Failed to seed {doc['section_title']}: {e}")
+
+if __name__ == "__main__":
+    ensure_ingested()