Create neural_searcher.py

neural_searcher.py

@@ -0,0 +1,42 @@
+from qdrant_client import QdrantClient
+from fastembed import SparseTextEmbedding
+from qdrant_client import QdrantClient, models
+from sentence_transformers import SentenceTransformer
+
+# from config import API_KEY,HOST,DENSE_MODEL,SPARSE_MODEL,DENSE_MODEL_SHORT,SPARSE_MODEL_SHORT
+class NeuralSearcher:
+
+    def __init__(self, collection_name):
+        self.collection_name = collection_name
+        self.dense_model = SentenceTransformer("djovak/embedic-small",device="cpu")
+        self.sparse_model = SparseTextEmbedding("Qdrant/bm25")
+        self.qdrant_client = QdrantClient("http://localhost:6333/",api_key="")
+    
+    def search(self, text: str):
+  
+        dense_query = self.dense_model.encode(text).tolist()
+        sparse_query = self.sparse_model.query_embed(text)
+        
+        # Use `vector` for search for closest vectors in the collection
+        search_result = self.qdrant_client.query_points(
+            collection_name= self.collection_name,
+            prefetch=[
+                models.Prefetch(
+                    query=dense_query,
+                    using="djovak/embedic-small",
+                    limit=5
+                ),
+                models.Prefetch(
+                    query=next(sparse_query).as_object(),
+                    using="Qdrant/bm25",
+                    limit=5
+                )
+            ],
+            query=models.FusionQuery(
+                fusion=models.Fusion.RRF
+            ),
+            limit = 9
+        ).points
+
+        payloads = [hit.payload for hit in search_result]
+        return payloads