Upload backend/hue_portal/core/search_ml.py with huggingface_hub

backend/hue_portal/core/search_ml.py

@@ -28,22 +28,40 @@ def expand_query_with_synonyms(query: str) -> List[str]:
     expanded = [query_normalized]
     
     try:
-        # Get all synonyms
-        synonyms = Synonym.objects.all()
+        # Limit to prevent too many expansions
+        max_expansions = 10
+        expansion_count = 0
+        
+        # Get all synonyms (limit to prevent too many DB queries)
+        synonyms = Synonym.objects.all()[:100]  # Limit to 100 synonyms
         for synonym in synonyms:
+            if expansion_count >= max_expansions:
+                break
+                
             keyword = normalize_text(synonym.keyword)
             alias = normalize_text(synonym.alias)
             
             # If query contains keyword, add alias
-            if keyword in query_normalized:
-                expanded.append(query_normalized.replace(keyword, alias))
+            if keyword and keyword in query_normalized:
+                new_query = query_normalized.replace(keyword, alias)
+                if new_query not in expanded:
+                    expanded.append(new_query)
+                    expansion_count += 1
+                    if expansion_count >= max_expansions:
+                        break
+            
             # If query contains alias, add keyword
-            if alias in query_normalized:
-                expanded.append(query_normalized.replace(alias, keyword))
+            if alias and alias in query_normalized:
+                new_query = query_normalized.replace(alias, keyword)
+                if new_query not in expanded:
+                    expanded.append(new_query)
+                    expansion_count += 1
+                    if expansion_count >= max_expansions:
+                        break
     except Exception:
         pass  # If Synonym table doesn't exist yet
     
-    return list(set(expanded))  # Remove duplicates
+    return list(set(expanded))[:10]  # Remove duplicates and limit to 10 variants
 
 
 def create_search_vector(text_fields: List[str]) -> str:
@@ -177,16 +195,41 @@ def search_with_ml(
     # Attempt PostgreSQL BM25 ranking first when available
     if connection.vendor == "postgresql" and hasattr(queryset.model, "tsv_body"):
         try:
-            expanded_queries = expand_query_with_synonyms(query)
-            combined_query = None
-            for q_variant in expanded_queries:
-                variant_query = SearchQuery(q_variant, config="simple")
-                combined_query = variant_query if combined_query is None else combined_query | variant_query
+            import sys
+            # Increase recursion limit for query expansion
+            old_limit = sys.getrecursionlimit()
+            try:
+                sys.setrecursionlimit(3000)  # Increase limit for query expansion
+                expanded_queries = expand_query_with_synonyms(query)
+                # Limit expanded queries to prevent too many variants
+                expanded_queries = expanded_queries[:5]  # Max 5 variants
+                
+                combined_query = None
+                for q_variant in expanded_queries:
+                    variant_query = SearchQuery(q_variant, config="simple")
+                    combined_query = variant_query if combined_query is None else combined_query | variant_query
 
-            if combined_query is not None:
+                if combined_query is not None:
+                    ranked_qs = (
+                        queryset
+                        .annotate(rank=SearchRank(F("tsv_body"), combined_query))
+                        .filter(rank__gt=0)
+                        .order_by("-rank")
+                    )
+                    results = list(ranked_qs[:top_k])
+                    if results:
+                        for obj in results:
+                            obj._ml_score = getattr(obj, "rank", 0.0)
+                        return results
+            finally:
+                sys.setrecursionlimit(old_limit)  # Restore original limit
+        except RecursionError as e:
+            # Fallback: use original query without expansion
+            try:
+                variant_query = SearchQuery(query, config="simple")
                 ranked_qs = (
                     queryset
-                    .annotate(rank=SearchRank(F("tsv_body"), combined_query))
+                    .annotate(rank=SearchRank(F("tsv_body"), variant_query))
                     .filter(rank__gt=0)
                     .order_by("-rank")
                 )
@@ -195,6 +238,8 @@ def search_with_ml(
                     for obj in results:
                         obj._ml_score = getattr(obj, "rank", 0.0)
                     return results
+            except Exception:
+                pass
         except Exception:
             # Fall through to ML-based search if any error occurs (e.g. missing extensions)
             pass
@@ -213,7 +258,14 @@ def search_with_ml(
     
     # Calculate similarity scores
     try:
-        scored_indices = calculate_similarity_scores(query, documents, top_k=top_k)
+        import sys
+        # Increase recursion limit for TF-IDF calculation
+        old_limit = sys.getrecursionlimit()
+        try:
+            sys.setrecursionlimit(3000)  # Increase limit for TF-IDF
+            scored_indices = calculate_similarity_scores(query, documents, top_k=top_k)
+        finally:
+            sys.setrecursionlimit(old_limit)  # Restore original limit
         
         # Filter by minimum score and get object IDs
         valid_indices = [idx for idx, score in scored_indices if score >= min_score]