Debug retriever and search

app/routes/ingest_routes.py

@@ -48,6 +48,7 @@ async def ingest_data(file: UploadFile = File(...)):
         
         # Load data from temporary file
         records = DataIngestionService.load_data(temp_file_path)
+        logger.debug(f"Loaded {len(records)} records from file")
         
         if not records:
             raise HTTPException(status_code=400, detail="No records found in file")

app/services/embeddings.py

@@ -19,8 +19,10 @@ class EmbeddingService:
     
     def embed_text(self, text: str) -> List[float]:
         """Generate embedding for a single text"""
-        embedding = self.model.encode(text, convert_to_numpy=True)
+        embedding = self.model.encode(text, convert_to_numpy=True, normalize_embeddings=True)
+        logger.debug(f"Generated embedding for text: {embedding}")
         return embedding.tolist()
+        
     
     def embed_batch(self, texts: List[str], batch_size: int = 32) -> List[List[float]]:
         """Generate embeddings for a batch of texts"""
@@ -29,12 +31,14 @@ class EmbeddingService:
             texts,
             batch_size=batch_size,
             show_progress_bar=True,
-            convert_to_numpy=True
+            convert_to_numpy=True,
+            normalize_embeddings=True
         )
         return embeddings.tolist()
     
     def get_dimension(self) -> int:
         """Return embedding dimension"""
+        logger.debug(f"Embedding dimension requested: {self.dimension}")
         return self.dimension
 
 # Global instance

app/services/retriever.py

@@ -16,12 +16,15 @@ class RetrieverService:
     
     def retrieve(self, query: str, top_k: int = None) -> List[Dict[str, Any]]:
         """Retrieve relevant documents for a query"""
+        logger.debug(f"top_k: {top_k}")
+        logger.debug(f"User Query: {query}")
         if top_k is None:
             top_k = settings.TOP_K
         
         # Generate query embedding
         logger.info(f"Retrieving documents for query: {query}")
         query_embedding = self.embedding_service.embed_text(query)
+        logger.debug(f"Embedded query: {query_embedding}")
         
         #FAISS
         results = self.vector_store.search(
@@ -30,6 +33,10 @@ class RetrieverService:
             score_threshold=settings.SCORE_THRESHOLD
         )
 
+        logger.debug(f"FAISS total vectors: {index.ntotal}")        
+        D, I =  self.vector_store.index.search(np.array([query_embedding]).astype("float32"), k=3)
+        logger.debug(f"Distances: {D}, Indices: {I}")
+
         #Qdrant
         # Search vector database
         # results = self.vector_store.search(

app/utils/vector_store.py

@@ -1,96 +0,0 @@
-"""Qdrant vector store service"""
-from qdrant_client import QdrantClient
-from qdrant_client.http import models
-from typing import List, Dict, Any
-from app.config import settings
-from app.utils.logger import setup_logger
-
-logger = setup_logger(__name__)
-
-class VectorStoreService:
-    """Manages Qdrant vector database operations"""
-    
-    def __init__(self):
-        """Initialize Qdrant client"""
-        logger.info(f"Connecting to Qdrant at {settings.QDRANT_URL}")
-        self.client = QdrantClient(
-            url=settings.QDRANT_URL,
-            api_key=settings.QDRANT_API_KEY if settings.QDRANT_API_KEY else None
-        )
-        self.collection_name = settings.QDRANT_COLLECTION_NAME
-    
-    def create_collection(self, vector_size: int):
-        """Create or recreate the collection"""
-        try:
-            # Delete if exists
-            self.client.delete_collection(collection_name=self.collection_name)
-            logger.info(f"Deleted existing collection: {self.collection_name}")
-        except:
-            pass
-        
-        # Create new collection
-        self.client.create_collection(
-            collection_name=self.collection_name,
-            vectors_config=models.VectorParams(
-                size=vector_size,
-                distance=models.Distance.COSINE
-            )
-        )
-        logger.info(f"Created collection: {self.collection_name}")
-    
-    def upsert_vectors(
-        self,
-        vectors: List[List[float]],
-        payloads: List[Dict[str, Any]]
-    ) -> int:
-        """Insert vectors with metadata"""
-        points = [
-            models.PointStruct(
-                id=idx,
-                vector=vector,
-                payload=payload
-            )
-            for idx, (vector, payload) in enumerate(zip(vectors, payloads))
-        ]
-        
-        self.client.upsert(
-            collection_name=self.collection_name,
-            points=points
-        )
-        
-        logger.info(f"Upserted {len(points)} vectors")
-        return len(points)
-    
-    def search(
-        self,
-        query_vector: List[float],
-        limit: int = 5,
-        score_threshold: float = 0.5
-    ) -> List[Dict[str, Any]]:
-        """Search for similar vectors"""
-        results = self.client.search(
-            collection_name=self.collection_name,
-            query_vector=query_vector,
-            limit=limit,
-            score_threshold=score_threshold
-        )
-        
-        return [
-            {
-                "id": result.id,
-                "score": result.score,
-                "payload": result.payload
-            }
-            for result in results
-        ]
-    
-    def get_collection_info(self) -> Dict[str, Any]:
-        """Get collection statistics"""
-        info = self.client.get_collection(collection_name=self.collection_name)
-        return {
-            "vectors_count": info.vectors_count,
-            "status": info.status
-        }
-
-# Global instance
-vector_store = VectorStoreService()