Update app.py

app.py

@@ -99,29 +99,35 @@ retriever = vectordb.as_retriever(
 def hybrid_search(query, top_k=5):
     vector_results = retriever.invoke(query)
 
-    bm_scores = bm25.get_scores(query.lower().split())
-    bm_ranked = sorted(zip(bm_scores, chunks), reverse=True)
-    bm_results = [doc for _, doc in bm_ranked[:top_k]]
+    tokenized_query = query.lower().split()
+    bm_scores = bm25.get_scores(tokenized_query)
+    
+    bm_ranked = sorted(zip(bm_scores, chunks), key=lambda x: x[0], reverse=True)
+    bm_results = [d for _, d in bm_ranked[:top_k]]
 
-    unique_docs = []
+    combined = vector_results + bm_results
+
+    # remove duplicates
     seen = set()
-    for doc in vector_results + bm_results:
-        key = (doc.metadata.get("doc_id"), doc.page_content[:60])
+    unique = []
+    for d in combined:
+        key = (d.metadata.get("doc_id"), d.page_content[:80])
         if key not in seen:
             seen.add(key)
-            unique_docs.append(doc)
+            unique.append(d)
 
-    if not unique_docs:
+    if not unique:
         return []
 
-    pairs = [(query, doc.page_content) for doc in unique_docs]
+    pairs = [(query, doc.page_content) for doc in unique]
     scores = reranker.predict(pairs)
-    ranked = sorted(zip(scores, unique_docs), reverse=True)[:top_k]
 
-    for score, doc in ranked:
-        doc.metadata["rerank_score"] = float(score)
+    ranked = sorted(zip(scores, unique), key=lambda x: x[0], reverse=True)[:top_k]
+    for s, doc in ranked:
+        doc.metadata["rerank_score"] = float(s)
+
+    return [doc for _, doc in ranked]
 
-    return [doc for score, doc in ranked]
 
 
 # ------------------ LLM ------------------ #