better hybrid search

app.py

@@ -2,7 +2,7 @@ import spaces
 import os
 from typing import List, Optional, Dict, Any
 import gradio as gr
-from qdrant_client import QdrantClient
+from qdrant_client import QdrantClient, models
 from sentence_transformers import SentenceTransformer
 from transformers import AutoModelForCausalLM, AutoTokenizer
 from peft import PeftModel
@@ -13,6 +13,8 @@ from dataclasses import dataclass
 
 warnings.filterwarnings("ignore")
 
+QDRANT_COLLECTION_NAME = os.getenv("QDRANT_COLLECTION_NAME")
+
 # Configuration with secure credential handling
 @dataclass
 class Config:
@@ -188,44 +190,68 @@ class FiftyOneAssistant:
         """Wrapper for GPU embedding function."""
         return get_embedding_gpu(text, config.embedding_model)
     
-    def hybrid_search(self, query_text: str, query_vector: List[float], limit: int = 5) -> List[Dict[str, Any]]:
-        """Perform hybrid search using global qdrant_client."""
-        if not qdrant_client:
-            raise ValueError("Qdrant client not initialized")
+    def hybrid_search(query_text: str, query_vector: List[float], limit: int = 3, text_boost: float = 0.3):
+        """
+        Hybrid search using only dense vectors + text filtering
+        """
         
-        # Dense vector search
-        dense_results = qdrant_client.query_points(
-            collection_name=config.qdrant_collection_name,
+        # Pure vector search for semantic similarity
+        vector_results = qdrant_client.query_points(
+            collection_name=QDRANT_COLLECTION_NAME,
             query=query_vector,
-            limit=limit * 2,
-            with_payload=True
+            limit=limit
         ).points
         
-        # Keyword scoring
-        query_words = set(word.lower() for word in query_text.split() if len(word) > 3)
+        # Vector search WITH text filter for exact matches
+        text_filtered_results = qdrant_client.query_points(
+            collection_name=QDRANT_COLLECTION_NAME,
+            query=query_vector,
+            query_filter=models.Filter(
+                should=[
+                    models.FieldCondition(
+                        key="query",
+                        match=models.MatchText(text=query_text)
+                    ),
+                    models.FieldCondition(
+                        key="response", 
+                        match=models.MatchText(text=query_text)
+                    )
+                ]
+            ),
+            limit=limit
+        ).points
         
-        scored_results = []
-        for result in dense_results:
-            payload = result.payload
-            doc_text = f"{payload.get('query', '')} {payload.get('response', '')}".lower()
-            
-            keyword_bonus = 0.0
-            if query_words:
-                doc_words = set(doc_text.split())
-                overlap = query_words.intersection(doc_words)
-                keyword_bonus = len(overlap) / len(query_words) * 0.2
-            
-            final_score = result.score + keyword_bonus
-            scored_results.append({'score': final_score, 'payload': payload})
+        # Create lookup for text matches
+        text_match_ids = {result.id for result in text_filtered_results}
+        
+        # Process all vector results and boost those with text matches
+        final_results = []
+        seen_ids = set()
+        
+        for result in vector_results:
+            if result.id not in seen_ids:
+                # Boost score if this item also has text matches
+                score = result.score
+                if result.id in text_match_ids:
+                    score = score + (score * text_boost)  # Proportional boost
+                
+                final_results.append({
+                    'id': result.id,
+                    'score': score,
+                    'payload': result.payload,
+                    'has_text_match': result.id in text_match_ids
+                })
+                seen_ids.add(result.id)
         
-        scored_results.sort(key=lambda x: x['score'], reverse=True)
-        return scored_results[:limit]
+        # Sort by boosted scores
+        final_results.sort(key=lambda x: x['score'], reverse=True)
+        return final_results[:limit]
     
     def get_context(self, user_query: str, top_k: int = 3) -> str:
         """Get relevant context."""
         try:
             query_vector = self.get_embedding(user_query)
-            results = self.hybrid_search(user_query, query_vector, top_k)
+            results = self.hybrid_search(user_query, query_vector)
             
             if not results:
                 return "No relevant content found"