Update advanced_rag.py

advanced_rag.py

@@ -187,20 +187,25 @@ Alternative queries (one per line):"""
         # Get Cross-Encoder scores
         ce_scores = self.cross_encoder.predict(pairs)
         
-        # Create reranked documents with new scores
+        # Normalize CE scores using sigmoid (convert logits to 0-1 range)
+        import math
+        def sigmoid(x):
+            return 1 / (1 + math.exp(-x))
+        
+        ce_scores_normalized = [sigmoid(float(score)) for score in ce_scores]
+        
+        # Create reranked documents with normalized scores
         reranked = []
-        for doc, ce_score in zip(documents, ce_scores):
-            # Combine CE score with original confidence (weighted)
-            combined_score = 0.7 * float(ce_score) + 0.3 * doc.confidence
-            
+        for doc, ce_score_norm in zip(documents, ce_scores_normalized):
+            # Use ONLY Cross-Encoder score (it's more accurate than cosine similarity)
             reranked.append(RetrievedDocument(
                 id=doc.id,
                 text=doc.text,
-                confidence=float(combined_score),
+                confidence=float(ce_score_norm),
                 metadata=doc.metadata
             ))
         
-        # Sort by new combined score
+        # Sort by Cross-Encoder score
         reranked.sort(key=lambda x: x.confidence, reverse=True)
         return reranked[:top_k]