Lenient thres

models/reranker.py

@@ -42,7 +42,7 @@ class MedicalReranker:
             'generic_health_site': 0.30
         }
         
-        # Irrelevant content patterns
+        # Irrelevant content patterns - more specific to avoid false positives
         self.irrelevant_patterns = [
             r'quiz|test|assessment|survey',
             r'homepage|main page|index',
@@ -54,7 +54,7 @@ class MedicalReranker:
             r'healthy-sleep/quiz',  # Sleep quiz example
         ]
     
-    def rerank_results(self, query: str, results: List[Dict], min_score: float = 0.15) -> List[Dict]:
+    def rerank_results(self, query: str, results: List[Dict], min_score: float = 0.05) -> List[Dict]:
         """Rerank search results based on medical relevance"""
         if not results:
             return []
@@ -105,10 +105,10 @@ class MedicalReranker:
                 logger.debug(f"Filtered irrelevant result: {url}")
                 continue
             
-            # Only skip if we have content and it's too short
+            # Only skip if we have content and it's extremely short
             # Don't filter based on content length if no content is available yet
-            if content and len(content) < 50:  # Reduced from 100 to 50
-                logger.debug(f"Filtered result with short content: {url}")
+            if content and len(content) < 20:  # Much more lenient - only filter very short content
+                logger.debug(f"Filtered result with very short content: {url}")
                 continue
             
             filtered.append(result)
@@ -123,8 +123,8 @@ class MedicalReranker:
             url = result.get('url', '')
             domain = self._extract_domain(url)
             
-            # Get domain score - be more lenient with unknown domains
-            domain_score = self.domain_scores.get(domain, 0.50)  # Increased default score
+            # Get domain score - be much more lenient with unknown domains
+            domain_score = self.domain_scores.get(domain, 0.70)  # Much higher default score
             
             # Boost score for medical-specific content
             title = result.get('title', '').lower()
@@ -243,8 +243,8 @@ class MedicalReranker:
             if query.lower() in title:
                 title_relevance = min(title_relevance + 0.3, 1.0)
             
-            # Update composite score - be more lenient
-            domain_score = result.get('domain_score', 0.5)  # Increased default
+            # Update composite score - be much more lenient
+            domain_score = result.get('domain_score', 0.7)  # Much higher default
             result['title_relevance'] = title_relevance
             result['composite_score'] = (domain_score * 0.3) + (title_relevance * 0.7)  # Favor title relevance