Updated vectorstore

app/main.py

@@ -119,6 +119,24 @@ async def health_check():
         uptime=uptime_str
     )
 
+# from app.rag.utils import cleanup_old_vectorstores
+
+# @asynccontextmanager
+# async def lifespan(app: FastAPI):
+#     global startup_time
+#     startup_time = time.time()
+#     logger.info("🚀 Starting %s v%s", settings.app_name, settings.app_version)
+#     logger.info("📊 Server will run on %s:%s", settings.host, settings.port)
+
+#     # Trigger cleanup on startup
+#     deleted = cleanup_old_vectorstores(days=7)
+#     logger.info("🧹 Cleanup complete. %s old sessions removed.", deleted)
+
+#     yield
+
+#     logger.info("📊 Shutting down %s", settings.app_name)
+
+
 @app.exception_handler(404)
 async def not_found_handler(request, exc):
     logger.warning("404 Not Found: %s %s", request.method, request.url.path)

app/mobile_usability/prompts.py

@@ -31,6 +31,10 @@ Formatting rules:
 - Each list may contain zero or more items, but critical items must appear in "high".
 - Ensure items are specific enough for a developer to action (mention affected selector(s) when possible).
 
+Important:
+- Respond with *only* a valid JSON object.
+- Do NOT include any commentary or explanation outside the JSON.
+
 {format_instructions}
 
 Use the following to guide prioritization:

app/page_speed/config.py

@@ -12,6 +12,11 @@ class Settings(BaseSettings):
     pagespeed_api_key: str
     gemini_api_key: str
 
+    # Qdrant (vector DB) connection (optional; if not set, QdrantClient will use defaults)
+    qdrant_url: str 
+    qdrant_api_key: str 
+    # Optional timeout (seconds) to use when creating clients or making calls
+    qdrant_timeout: int = 60
 
     # ───────────────────────────────────────────────────────────────────────────
     # Chat & RAG Configuration
@@ -30,18 +35,18 @@ class Settings(BaseSettings):
 
     @property
     def mongo_uri(self) -> str:
-        pw = quote_plus(self.mongo_password)
-        return (
-            f"mongodb+srv://{self.mongo_user}:{pw}@{self.mongo_host}/"
-            f"{self.mongo_db}?retryWrites=true&w=majority&ssl=true"
-        )
+        # pw = quote_plus(self.mongo_password)
+        # return (
+        #     f"mongodb+srv://{self.mongo_user}:{pw}@{self.mongo_host}/"
+        #     f"{self.mongo_db}?retryWrites=true&w=majority&ssl=true"
+        # )
 
 
     # ───────────────────────────────────────────────────────────────────────────
     # local MongoDB Connection
     # ───────────────────────────────────────────────────────────────────────────
      
-        #return f"mongodb://localhost:27017/{self.mongo_db}"
+        return f"mongodb://localhost:27017/{self.mongo_db}"
     
 
     # ───────────────────────────────────────────────────────────────────────────

app/rag/chat_history.py

@@ -1,12 +1,10 @@
-import os
 import time
 from typing import List, Dict, Any
 from pymongo import ReturnDocument
 
 from app.page_speed.config import settings
-from .db import mongo_client, chat_collection_name
+from .db import mongo_client, chat_collection_name, qdrant_client
 from .embeddings import get_llm
-from .utils import get_vectorstore_path  # make sure this util is available
 from langchain.prompts import ChatPromptTemplate
 from .logging_config import logger
 
@@ -21,6 +19,7 @@ summarization_prompt = ChatPromptTemplate.from_messages([
     ("human", "{chat_history}")
 ])
 
+
 class ChatHistoryManager:
     @staticmethod
     def create_session(chat_id: str) -> None:
@@ -82,13 +81,13 @@ class ChatHistoryManager:
         return True
 
     @staticmethod
-    def vectorstore_exists(user_id: str) -> bool:
+    def vectorstore_exists(collection_name: str) -> bool:
         """
-        Check if a vectorstore directory already exists for this user.
+        Check if a Qdrant collection exists instead of local FAISS path.
         """
-        path = get_vectorstore_path(user_id)
-        exists = os.path.isdir(path)
-        logger.debug("Vectorstore path %s exists: %s", path, exists)
+        collections = qdrant_client.get_collections().collections
+        exists = any(c.name == collection_name for c in collections)
+        logger.debug("Qdrant collection %s exists: %s", collection_name, exists)
         return exists
 
     @staticmethod

app/rag/db.py

@@ -1,38 +1,57 @@
-# db.py
-from pymongo import MongoClient
-from app.page_speed.config import settings
+# # db.py
+# from pymongo import MongoClient
+# from app.page_speed.config import settings
+#from qdrant_client import QdrantClient
 
-# ──────────────────────────────────────────────────────────────────────────────
-# MongoDB Initialization
-# ──────────────────────────────────────────────────────────────────────────────
+# # ──────────────────────────────────────────────────────────────────────────────
+# # MongoDB Initialization
+# # ──────────────────────────────────────────────────────────────────────────────
 
-# Connect to MongoDB using the URI from settings
-mongo_client = MongoClient(settings.mongo_uri)
+# # Connect to MongoDB using the URI from settings
+# mongo_client = MongoClient(settings.mongo_uri)
 
-# Use the renamed settings attributes
-mongo_db = mongo_client[settings.mongo_db]
+# # Use the renamed settings attributes
+# mongo_db = mongo_client[settings.mongo_db]
 
-# Collection to store metadata that maps user_id → vectorstore_path
-vectorstore_meta_coll = mongo_db["vectorstore_metadata"]
+# # Collection to store metadata that maps user_id → vectorstore_path
+# vectorstore_meta_coll = mongo_db["vectorstore_metadata"]
 
-# Name of the collection that MongoDBChatMessageHistory will write to
-chat_collection_name = settings.mongo_collection
+# # Name of the collection that MongoDBChatMessageHistory will write to
+# chat_collection_name = settings.mongo_collection
 
+# # ─────────────────────────────────────────────
+# # Qdrant Setup
+# # ─────────────────────────────────────────────
+# # If Qdrant is running locally
+# qdrant_client = QdrantClient(
+#     url=settings.qdrant_url,  # e.g. "http://localhost:6333"
+#     api_key=settings.qdrant_api_key or None
+# )
 
-# # ____________________________________________________________
-# #Local MongoDB Connection
-# # ____________________________________________________________
+# ____________________________________________________________
+#Local MongoDB Connection
+# ____________________________________________________________
 
-# # db.py
-# from pymongo import MongoClient
-# from app.page_speed.config import settings
+# db.py
+from pymongo import MongoClient
+from app.page_speed.config import settings
+from qdrant_client import QdrantClient
 
-# # Always connect to local MongoDB
-# mongo_client = MongoClient("mongodb://localhost:27017/")
+# Always connect to local MongoDB
+mongo_client = MongoClient("mongodb://localhost:27017/")
 
-# # Select the database from settings
-# mongo_db = mongo_client[settings.mongo_db]
+# Select the database from settings
+mongo_db = mongo_client[settings.mongo_db]
 
-# # Collections
-# vectorstore_meta_coll = mongo_db["vectorstore_metadata"]
-# chat_collection_name = settings.mongo_collection
+# Collections
+vectorstore_meta_coll = mongo_db["vectorstore_metadata"]
+chat_collection_name = settings.mongo_collection
+
+# ─────────────────────────────────────────────
+# Qdrant Setup
+# ─────────────────────────────────────────────
+# If Qdrant is running locally
+qdrant_client = QdrantClient(
+    url=settings.qdrant_url,  # e.g. "http://localhost:6333"
+    api_key=settings.qdrant_api_key or None
+)

app/rag/routes.py

@@ -1,12 +1,16 @@
+# app/rag/routes.py
+
 import os
+import json
 import uuid
+import time
+from typing import List, Optional, Iterable
+
 from fastapi import APIRouter, HTTPException, Path, Query
 
 from .schemas import SetupRequest, ChatRequest, SetupResponse, ChatResponse
 from .utils import (
     get_vectorstore_path,
-    text_splitter,
-    embeddings,
     save_vectorstore_to_disk,
     upsert_vectorstore_metadata,
     get_vectorstore_metadata,
@@ -15,8 +19,49 @@ from .utils import (
 from .chat_history import ChatHistoryManager
 from .logging_config import logger
 
+from qdrant_client import QdrantClient
+from qdrant_client.models import VectorParams, PointStruct, Distance
+from app.page_speed.config import settings
+from .embeddings import embeddings, text_splitter  # kept here for ingestion
+
 router = APIRouter(prefix="/rag", tags=["rag"])
 
+
+def _get_embeddings_for_texts(texts: List[str]) -> List[List[float]]:
+    """
+    Try common embedding API names (embed_documents, embed_texts, embed).
+    Falls back to calling embed_query per text (slower).
+    """
+    if not texts:
+        return []
+
+    # Preferred bulk API
+    for attr in ("embed_documents", "embed_texts", "embed_batch", "embed"):
+        fn = getattr(embeddings, attr, None)
+        if callable(fn):
+            try:
+                return fn(texts)
+            except Exception:
+                logger.debug("Embedding method %s failed; trying next option", attr, exc_info=True)
+
+    # Fallback: try single-item embedding function repeatedly
+    single_fn = getattr(embeddings, "embed_query", None) or getattr(embeddings, "embed", None)
+    if callable(single_fn):
+        vecs = []
+        for t in texts:
+            vec = single_fn(t)
+            if isinstance(vec, dict) and "embedding" in vec:
+                vecs.append(vec["embedding"])
+            else:
+                vecs.append(vec)
+        return vecs
+
+    raise RuntimeError(
+        "Embeddings object does not expose a supported embedding method "
+        "(embed_documents/embed_texts/embed_query)."
+    )
+
+
 @router.post("/initialization/{onboarding_id}/{doc_type}", response_model=SetupResponse)
 async def setup_rag_session(
     onboarding_id: str = Path(..., description="Unique onboarding identifier"),
@@ -25,34 +70,35 @@ async def setup_rag_session(
 ):
     """
     Ingest documents under a specific document type and create a chat session.
-    - If vectorstore exists for onboarding_id and doc_type, skip ingestion.
+    - If vectorstore metadata exists for onboarding_id and doc_type in MongoDB, skip ingestion.
     - Always create a new chat_id for this session.
+    NOTE: This implementation does NOT create or rely on any local files on disk for metadata.
     """
-    vectorstore_path = get_vectorstore_path(onboarding_id, doc_type)
-
-    # Existing vectorstore
-    if os.path.isdir(os.path.join(vectorstore_path, "faiss_index")):
+    # Use DB metadata instead of local filesystem marker
+    existing_meta = get_vectorstore_metadata(onboarding_id, doc_type)
+    if existing_meta:
         logger.info(
-            "Vectorstore exists for onboarding_id=%s, doc_type=%s; skipping ingestion",
+            "Vectorstore metadata exists for onboarding_id=%s, doc_type=%s; skipping ingestion",
             onboarding_id, doc_type
         )
-        metadata = get_vectorstore_metadata(onboarding_id, doc_type)
+        metadata = existing_meta
         if metadata and metadata.get("chat_id"):
             chat_id = metadata["chat_id"]
         else:
             chat_id = str(uuid.uuid4())
             ChatHistoryManager.create_session(chat_id)
-            upsert_vectorstore_metadata(onboarding_id, doc_type, vectorstore_path, chat_id)
+            # ensure DB has chat_id
+            upsert_vectorstore_metadata(onboarding_id, doc_type, metadata.get("vectorstore_path"), chat_id, metadata.get("collection_name"))
         return SetupResponse(
             success=True,
-            message="RAG setup completed with existing vectorstore.",
+            message="RAG setup completed with existing vectorstore metadata.",
             onboarding_id=onboarding_id,
             doc_type=doc_type,
             chat_id=chat_id,
-            vectorstore_path=vectorstore_path
+            vectorstore_path=metadata.get("vectorstore_path")
         )
 
-    # New ingestion
+    # New ingestion flow
     if not body.documents:
         logger.error(
             "Missing documents for onboarding_id=%s, doc_type=%s",
@@ -63,14 +109,123 @@ async def setup_rag_session(
     # Create session and ingest
     chat_id = str(uuid.uuid4())
     ChatHistoryManager.create_session(chat_id)
+
     all_text = "\n\n".join(body.documents)
     text_chunks = text_splitter.split_text(all_text)
-    vs = __import__("langchain_community.vectorstores").vectorstores.FAISS.from_texts(
-        texts=text_chunks,
-        embedding=embeddings
+
+    # Build Qdrant client from settings (with timeout + optional prefer_grpc)
+    client_kwargs = {}
+    if getattr(settings, "qdrant_url", None):
+        client_kwargs["url"] = settings.qdrant_url
+    if getattr(settings, "qdrant_api_key", None):
+        client_kwargs["api_key"] = settings.qdrant_api_key
+
+    # sensible defaults; override via app config
+    qdrant_timeout = getattr(settings, "qdrant_timeout", 60)        # seconds (default 60)
+    prefer_grpc = getattr(settings, "qdrant_prefer_grpc", False)    # set True to use gRPC if available
+
+    try:
+        if client_kwargs:
+            qdrant_client = QdrantClient(**client_kwargs, timeout=qdrant_timeout, prefer_grpc=prefer_grpc)
+        else:
+            qdrant_client = QdrantClient(timeout=qdrant_timeout, prefer_grpc=prefer_grpc)
+    except TypeError as e:
+        logger.exception("Failed to instantiate QdrantClient: %s", e)
+        raise HTTPException(status_code=500, detail=f"Failed to construct Qdrant client: {e}")
+
+    # Deterministic collection name for each onboarding/doc_type
+    collection_name = f"vs_{onboarding_id}_{doc_type}"
+
+    # --------------------------
+    # INGEST: compute embeddings
+    # --------------------------
+    try:
+        vectors = _get_embeddings_for_texts(text_chunks)
+    except Exception as e:
+        logger.exception("Failed to compute embeddings: %s", e)
+        raise HTTPException(status_code=500, detail=f"Embedding error: {e}")
+
+    if not vectors or len(vectors) != len(text_chunks):
+        logger.error("Embeddings length mismatch: vectors=%s texts=%s", len(vectors), len(text_chunks))
+        raise HTTPException(status_code=500, detail="Embedding generation failed or returned unexpected shape.")
+
+    vector_size = len(vectors[0])
+    if vector_size == 0:
+        raise HTTPException(status_code=500, detail="Embedding returned empty vectors")
+
+    # Recreate collection (idempotent for onboarding+doc_type)
+    try:
+        qdrant_client.recreate_collection(
+            collection_name=collection_name,
+            vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE)
+        )
+    except Exception as e:
+        logger.exception("Failed to create/recreate qdrant collection '%s': %s", collection_name, e)
+        raise HTTPException(status_code=500, detail=f"Failed to create qdrant collection: {e}")
+
+    # Helper: safe upsert with retries/backoff
+    def safe_upsert(client: QdrantClient, collection_name: str, points: Iterable[PointStruct], max_retries: int = 3):
+        attempt = 0
+        backoff = 1.0
+        last_exc: Optional[Exception] = None
+        while attempt < max_retries:
+            try:
+                client.upsert(collection_name=collection_name, points=points)
+                return
+            except Exception as exc:
+                last_exc = exc
+                attempt += 1
+                logger.warning("Qdrant upsert attempt %d/%d failed: %s", attempt, max_retries, exc)
+                if attempt >= max_retries:
+                    logger.exception("Qdrant upsert failed after %d attempts", max_retries)
+                    raise
+                # exponential backoff
+                time.sleep(backoff)
+                backoff *= 2.0
+        # if loop finishes without returning, raise last exception
+        if last_exc:
+            raise last_exc
+
+    # Upsert points in smaller batches and use safe_upsert
+    batch_size = getattr(settings, "qdrant_upsert_batch_size", 64)  # smaller default batch size
+    points_batch: List[PointStruct] = []
+    try:
+        for i, (vec, txt) in enumerate(zip(vectors, text_chunks)):
+            payload = {"text": txt}
+            # Use UUID string for id to avoid collisions across sessions
+            point_id = str(uuid.uuid4())
+            point = PointStruct(id=point_id, vector=vec, payload=payload)
+            points_batch.append(point)
+
+            if len(points_batch) >= batch_size:
+                logger.debug("Upserting batch of %d points to collection %s", len(points_batch), collection_name)
+                safe_upsert(qdrant_client, collection_name, points_batch)
+                points_batch = []
+
+        # final flush
+        if points_batch:
+            logger.debug("Upserting final batch of %d points to collection %s", len(points_batch), collection_name)
+            safe_upsert(qdrant_client, collection_name, points_batch)
+    except Exception as e:
+        logger.exception("Failed to upsert points into qdrant: %s", e)
+        raise HTTPException(status_code=500, detail=f"Failed to upsert points into Qdrant: {e}")
+
+    # Create an in-application "vectorstore_path" (URI-style) and store metadata in DB
+    vs_path = save_vectorstore_to_disk(
+        onboarding_id,
+        doc_type,
+        collection_name,
+        getattr(settings, "qdrant_url", None),
+        getattr(settings, "qdrant_api_key", None)
+    )
+    # Persist metadata into MongoDB (no local disk involved)
+    # Persist extra metadata fields so retrieval can use same connection details (if desired)
+    upsert_vectorstore_metadata(onboarding_id, doc_type, vs_path, chat_id, collection_name)
+
+    logger.info(
+        "Created Qdrant collection %s for %s/%s (points=%d)",
+        collection_name, onboarding_id, doc_type, len(text_chunks)
     )
-    vs_path = save_vectorstore_to_disk(vs, onboarding_id, doc_type)
-    upsert_vectorstore_metadata(onboarding_id, doc_type, vs_path, chat_id)
 
     return SetupResponse(
         success=True,
@@ -81,6 +236,7 @@ async def setup_rag_session(
         vectorstore_path=vs_path
     )
 
+
 @router.post("/chat/{onboarding_id}/{doc_type}/{chat_id}", response_model=ChatResponse)
 async def chat_with_user(
     onboarding_id: str = Path(...),
@@ -92,21 +248,25 @@ async def chat_with_user(
     """
     Chat endpoint using a specific document-type vectorstore.
     """
-    vectorstore_path = get_vectorstore_path(onboarding_id, doc_type)
-    if not os.path.isdir(os.path.join(vectorstore_path, "faiss_index")):
-        raise HTTPException(status_code=400, detail="Vectorstore not found; run initialization first.")
+    # Use DB metadata instead of local filesystem marker
+    metadata = get_vectorstore_metadata(onboarding_id, doc_type)
+    if not metadata:
+        raise HTTPException(status_code=400, detail="Vectorstore metadata not found; run initialization first.")
 
     if not ChatHistoryManager.chat_exists(chat_id):
         raise HTTPException(status_code=404, detail=f"Chat session {chat_id} not found.")
 
-    question = body.question.strip()
+    question = (body.question or "").strip()
+    if not question:
+        raise HTTPException(status_code=400, detail="Question cannot be empty.")
+
     ChatHistoryManager.summarize_if_needed(chat_id, threshold=10)
     ChatHistoryManager.add_message(chat_id, role="human", content=question)
 
     chain = build_rag_chain(onboarding_id, doc_type, chat_id, prompt_type)
     history = ChatHistoryManager.get_messages(chat_id)
     result = chain.invoke({"question": question, "chat_history": history})
-    answer = result.get("answer") or result.get("output_text")
+    answer = result.get("answer") or result.get("output_text") or ""
     ChatHistoryManager.add_message(chat_id, role="ai", content=answer)
 
     return ChatResponse(
@@ -116,4 +276,4 @@ async def chat_with_user(
         chat_id=chat_id,
         onboarding_id=onboarding_id,
         doc_type=doc_type
-    )
+    )

app/rag/utils.py

@@ -1,11 +1,22 @@
+# app/rag/utils.py
+
 import os
-from typing import Optional, Dict, Any
+import json
+from typing import Optional, Dict, Any, List
+from datetime import datetime
+
 from fastapi import HTTPException
 
-from langchain_community.vectorstores import FAISS
+from qdrant_client import QdrantClient
+from qdrant_client.http import models as qdrant_models
+
 from langchain_mongodb.chat_message_histories import MongoDBChatMessageHistory
 from langchain.memory import ConversationBufferMemory
 from langchain.chains import ConversationalRetrievalChain
+from langchain_core.retrievers import BaseRetriever
+from langchain_core.documents import Document
+
+from pydantic import ConfigDict  # Pydantic v2 config for BaseModel-based classes
 
 from app.page_speed.config import settings
 from .db import vectorstore_meta_coll, chat_collection_name
@@ -20,46 +31,160 @@ from .prompt_library import (
     mobile_usability_prompt
 )
 
-# 1. Path with doc_type
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Paths & metadata helpers (diskless)
+# ──────────────────────────────────────────────────────────────────────────────
+
 def get_vectorstore_path(onboarding_id: str, doc_type: str) -> str:
     """
-    Returns './vectorstores/{onboarding_id}/{doc_type}'.
+    Returns a non-disk URI-like path for a vectorstore.
+    Example: 'qdrant://<onboarding_id>/<doc_type>'
+    This avoids creating a local folder while preserving a string that identifies
+    the logical vectorstore for other components and logs.
     """
-    base_dir = settings.vectorstore_base_path
-    return os.path.join(base_dir, onboarding_id, doc_type)
+    return f"qdrant://{onboarding_id}/{doc_type}"
+
 
-# 2. Save to disk under doc_type
-def save_vectorstore_to_disk(vectorstore: FAISS, onboarding_id: str, doc_type: str) -> str:
+def save_vectorstore_to_disk(
+    onboarding_id: str,
+    doc_type: str,
+    collection_name: str,
+    qdrant_url: Optional[str],
+    qdrant_api_key: Optional[str]
+) -> str:
     """
-    Save under './vectorstores/{onboarding_id}/{doc_type}/faiss_index'.
+    Previously this created a small local marker file with Qdrant connection details.
+    In the diskless version we simply return a logical vectorstore path (URI-style).
+    Persisting of metadata is done via `upsert_vectorstore_metadata`.
     """
-    vs_dir = get_vectorstore_path(onboarding_id, doc_type)
-    faiss_index_path = os.path.join(vs_dir, "faiss_index")
-    os.makedirs(faiss_index_path, exist_ok=True)
-    vectorstore.save_local(folder_path=faiss_index_path)
-    return faiss_index_path
+    vs_path = get_vectorstore_path(onboarding_id, doc_type)
+    return vs_path
+
 
-# 3. Metadata now includes doc_type
 def upsert_vectorstore_metadata(
     onboarding_id: str,
     doc_type: str,
     vectorstore_path: str,
-    chat_id: str
+    chat_id: str,
+    collection_name: Optional[str] = None,
+    qdrant_url: Optional[str] = None,
+    qdrant_api_key: Optional[str] = None
 ) -> None:
+    """
+    Store metadata in MongoDB. Saves useful fields to allow build_rag_chain to
+    reconstruct a working Qdrant client later.
+    """
+    update = {
+        "onboarding_id": onboarding_id,
+        "doc_type": doc_type,
+        "vectorstore_path": vectorstore_path,
+        "chat_id": chat_id,
+        "updated_at": datetime.utcnow(),
+    }
+    if collection_name:
+        update["collection_name"] = collection_name
+    if qdrant_url:
+        update["qdrant_url"] = qdrant_url
+    if qdrant_api_key:
+        update["qdrant_api_key"] = qdrant_api_key
+
+    # Upsert the document
     vectorstore_meta_coll.update_one(
         {"onboarding_id": onboarding_id, "doc_type": doc_type},
-        {"$set": {"vectorstore_path": vectorstore_path, "chat_id": chat_id}},
+        {"$set": update, "$setOnInsert": {"created_at": datetime.utcnow()}},
         upsert=True
     )
+    logger.debug("Upserted vectorstore metadata for %s/%s into Mongo", onboarding_id, doc_type)
 
 
 def get_vectorstore_metadata(
     onboarding_id: str,
     doc_type: str
 ) -> Optional[Dict[str, Any]]:
-    return vectorstore_meta_coll.find_one({"onboarding_id": onboarding_id, "doc_type": doc_type})
+    """
+    Read vectorstore metadata from MongoDB (no local files).
+    """
+    meta = vectorstore_meta_coll.find_one({"onboarding_id": onboarding_id, "doc_type": doc_type})
+    if meta:
+        # convert ObjectId or other non-serializable fields if necessary
+        return meta
+    return None
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Qdrant Retriever (pure Qdrant, Pydantic v2-compatible)
+# ──────────────────────────────────────────────────────────────────────────────
+
+class QdrantTextRetriever(BaseRetriever):
+    """
+    Minimal retriever that queries Qdrant directly and returns LangChain Documents.
+    Assumes payload stores the raw chunk under key 'text'.
+    """
+
+    client: QdrantClient
+    collection_name: str
+    k: int = 5
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    def _get_relevant_documents(self, query: str, *, run_manager=None) -> List[Document]:
+        # Embed the query. Try multiple attribute names safely.
+        query_vec = None
+        for attr in ("embed_query", "embed_documents", "embed_texts", "embed"):
+            fn = getattr(embeddings, attr, None)
+            if callable(fn):
+                try:
+                    if attr == "embed_query":
+                        query_vec = fn(query)
+                    else:
+                        q_res = fn([query])
+                        if isinstance(q_res, list) and q_res:
+                            query_vec = q_res[0]
+                        else:
+                            query_vec = q_res
+                    break
+                except Exception:
+                    continue
+        if query_vec is None:
+            raise RuntimeError("No usable embedding function available on embeddings object.")
+
+        # If embedding helpers return dicts
+        if isinstance(query_vec, dict) and "embedding" in query_vec:
+            query_vec = query_vec["embedding"]
 
-# 4. Build chain now takes doc_type
+        # Search Qdrant
+        results = self.client.search(
+            collection_name=self.collection_name,
+            query_vector=query_vec,
+            limit=self.k
+        )
+
+        docs: List[Document] = []
+        for r in results:
+            payload = r.payload or {}
+            text = payload.get("text")
+            if not isinstance(text, str):
+                logger.warning(
+                    "Qdrant payload missing 'text' or not a string; skipping. Payload: %s",
+                    payload
+                )
+                continue
+
+            metadata = {k: v for k, v in payload.items() if k != "text"}
+            metadata["score"] = r.score
+
+            docs.append(Document(page_content=text, metadata=metadata))
+
+        return docs
+
+    async def _aget_relevant_documents(self, query: str, *, run_manager=None) -> List[Document]:
+        # For simplicity, use sync path
+        return self._get_relevant_documents(query, run_manager=run_manager)
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Build RAG chain (pure Qdrant), using DB metadata (no local files)
+# ──────────────────────────────────────────────────────────────────────────────
 
 def build_rag_chain(
     onboarding_id: str,
@@ -67,14 +192,112 @@ def build_rag_chain(
     chat_id: str,
     prompt_type: str
 ) -> ConversationalRetrievalChain:
-    # Load index
-    vs_path = get_vectorstore_path(onboarding_id, doc_type)
-    faiss_vs = FAISS.load_local(
-        folder_path=os.path.join(vs_path, "faiss_index"),
-        embeddings=embeddings,
-        allow_dangerous_deserialization=True
-    )
-    retriever = faiss_vs.as_retriever(search_kwargs={"k": 5})
+    """
+    Builds a ConversationalRetrievalChain using pure Qdrant as backend.
+    Loads connection details from the MongoDB metadata collection instead of a file.
+    If metadata is missing, tries to detect an existing Qdrant collection named
+    'vs_{onboarding_id}_{doc_type}' and auto-registers it in Mongo.
+    """
+    meta = get_vectorstore_metadata(onboarding_id, doc_type)
+
+    # If metadata missing — attempt a Qdrant-side fallback detection
+    if not meta:
+        logger.warning("Vectorstore metadata not found for %s/%s in Mongo; attempting Qdrant fallback detection", onboarding_id, doc_type)
+
+        # Build a Qdrant client from global settings to detect existing collection
+        qdrant_url = getattr(settings, "qdrant_url", None)
+        qdrant_api_key = getattr(settings, "qdrant_api_key", None)
+        client_kwargs = {}
+        if qdrant_url:
+            client_kwargs["url"] = qdrant_url
+        if qdrant_api_key:
+            client_kwargs["api_key"] = qdrant_api_key
+
+        qdrant_timeout = getattr(settings, "qdrant_timeout", 60)
+        prefer_grpc = getattr(settings, "qdrant_prefer_grpc", False)
+
+        try:
+            if client_kwargs:
+                qdrant_client = QdrantClient(**client_kwargs, timeout=qdrant_timeout, prefer_grpc=prefer_grpc)
+            else:
+                qdrant_client = QdrantClient(timeout=qdrant_timeout, prefer_grpc=prefer_grpc)
+        except Exception as e:
+            logger.exception("Failed to create Qdrant client during fallback detection: %s", e)
+            raise HTTPException(status_code=500, detail="Vectorstore metadata not found and failed to connect to Qdrant for fallback detection.")
+
+        guessed_collection = f"vs_{onboarding_id}_{doc_type}"
+        try:
+            # get_collection raises if not present; get_collections returns list
+            info = None
+            try:
+                info = qdrant_client.get_collection(collection_name=guessed_collection)
+            except Exception:
+                # try listing collections (less strict)
+                collections_info = qdrant_client.get_collections()
+                # get_collections returns a dict-like structure; search names
+                found = False
+                for c in collections_info.get("collections", []) if isinstance(collections_info, dict) else collections_info:
+                    name = c.get("name") if isinstance(c, dict) else getattr(c, "name", None)
+                    if name == guessed_collection:
+                        found = True
+                        break
+                if not found:
+                    info = None
+                else:
+                    info = {"name": guessed_collection}
+
+            if info:
+                logger.info("Detected existing Qdrant collection '%s' via fallback; auto-registering metadata in Mongo", guessed_collection)
+                # auto-register minimal metadata so chat can proceed
+                vs_path = get_vectorstore_path(onboarding_id, doc_type)
+                # we don't have a chat_id to store here; store empty string and let setup create chat sessions later
+                upsert_vectorstore_metadata(onboarding_id, doc_type, vs_path, chat_id="", collection_name=guessed_collection, qdrant_url=qdrant_url, qdrant_api_key=qdrant_api_key)
+                meta = get_vectorstore_metadata(onboarding_id, doc_type)
+            else:
+                logger.info("Qdrant fallback detection found no collection named '%s'", guessed_collection)
+        except Exception as e:
+            logger.exception("Error while checking Qdrant collections for fallback detection: %s", e)
+            # continue; meta still None and we'll raise below
+
+    if not meta:
+        # Final: helpful error message with actionable next steps
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                "Vectorstore metadata not found; run initialization first. "
+                "Call POST /rag/initialization/{onboarding_id}/{doc_type} with documents to ingest. "
+                "If you already initialized, check server logs for ingestion errors and verify Mongo collection "
+                "'vectorstore_meta_coll' contains the record for this onboarding/doc_type."
+            )
+        )
+
+    collection_name = meta.get("collection_name")
+    if not collection_name:
+        raise HTTPException(status_code=500, detail="Qdrant collection name missing in metadata.")
+
+    # Prefer values from marker; fall back to app settings if needed
+    qdrant_url = meta.get("qdrant_url") or getattr(settings, "qdrant_url", None)
+    qdrant_api_key = meta.get("qdrant_api_key") or getattr(settings, "qdrant_api_key", None)
+
+    client_kwargs = {}
+    if qdrant_url:
+        client_kwargs["url"] = qdrant_url
+    if qdrant_api_key:
+        client_kwargs["api_key"] = qdrant_api_key
+
+    qdrant_timeout = getattr(settings, "qdrant_timeout", 60)
+    prefer_grpc = getattr(settings, "qdrant_prefer_grpc", False)
+
+    try:
+        if client_kwargs:
+            qdrant_client = QdrantClient(**client_kwargs, timeout=qdrant_timeout, prefer_grpc=prefer_grpc)
+        else:
+            qdrant_client = QdrantClient(timeout=qdrant_timeout, prefer_grpc=prefer_grpc)
+    except Exception as e:
+        logger.exception("Failed to construct Qdrant client for retrieval: %s", e)
+        raise HTTPException(status_code=500, detail=f"Failed to connect to Qdrant: {e}")
+
+    retriever = QdrantTextRetriever(client=qdrant_client, collection_name=collection_name, k=5)
 
     # History & memory
     chat_history = MongoDBChatMessageHistory(
@@ -85,10 +308,12 @@ def build_rag_chain(
     )
     memory = ConversationBufferMemory(
         memory_key="chat_history",
-        chat_history=chat_history
+        chat_memory=chat_history,
+        return_messages=True,
     )
 
     llm = get_llm()
+
     # Choose prompt
     if prompt_type == "page_speed":
         user_prompt = page_speed_prompt

app/seo/prompts.py

@@ -19,6 +19,10 @@ Return *only* a JSON object that has a single top-level key, `priority_suggestio
 
 Each list item must be a **plain-English sentence**, prefixed with its SEO category tag (e.g. `[On-Page]` or `[Schema]`), and suffixed with `(Effort Level: high|medium|low)`.
 
+Important:
+- Respond with *only* a valid JSON object.
+- Do NOT include any commentary or explanation outside the JSON.
+
 {format_instructions}
 
 Performance Report:

app/uiux/prompts.py

@@ -21,6 +21,10 @@ Requirements:
 6. Ensure the output is strictly JSON—no additional text, comments, or keys.
 7. Validate JSON syntax: keys and strings must be enclosed in double quotes.
 
+Important:
+- Respond with *only* a valid JSON object.
+- Do NOT include any commentary or explanation outside the JSON.
+
 {format_instructions}
 
 Input Report Data:

requirements.txt

@@ -7,7 +7,7 @@ pydantic
 pydantic_settings
 langchain_groq 
 langchain_community 
-faiss-cpu 
 pymongo 
 langchain-mongodb
 langchain_google_genai
+qdrant-client