Spacy for getting search terms

app.py

@@ -8,6 +8,11 @@ from typing import Optional, List, Dict, Any
 import logging
 import re
 
+
+import spacy
+from rake_nltk import Rake
+import nltk
+
 # Configure logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -48,6 +53,63 @@ classifier_pipeline = None
 qa_pipeline = None
 summarization_pipeline = None
 ner_pipeline = None
+nlp = spacy.load("en_core_web_sm")
+rake = Rake()
+
+
+############# SPACY TEST
+
+def extract_search_terms(text: str) -> List[str]:
+    """Extract enhanced search terms using combined NER, syntax, and keywords"""
+    doc = nlp(text)
+    
+    # 1. Extract named entities
+    entities = [ent.text for ent in doc.ents]
+    
+    # 2. Extract noun phrases through syntactic analysis
+    noun_phrases = list(doc.noun_chunks)
+    
+    # 3. Extract question focus using dependency parsing
+    focus_phrase = extract_focus_phrase(doc)
+    
+    # 4. Get keywords using RAKE
+    rake.extract_keywords_from_text(text)
+    keywords = rake.get_ranked_phrases()[:3]  # Top 3 keywords
+    
+    # Combine and filter terms
+    terms = entities + [np.text for np in noun_phrases] + keywords
+    if focus_phrase:
+        terms.append(focus_phrase)
+    
+    # Clean and deduplicate
+    return clean_terms(terms)
+
+def extract_focus_phrase(doc) -> str:
+    """Extract main question focus using dependency parse"""
+    for token in doc:
+        if token.dep_ == "ROOT":
+            for child in token.children:
+                if child.dep_ in ("attr", "nsubj", "dobj"):
+                    return " ".join([t.text for t in child.subtree])
+    return ""
+
+def clean_terms(terms: List[str]) -> List[str]:
+    """Remove duplicates and irrelevant terms"""
+    # Remove stopwords and single characters
+    cleaned = [
+        t for t in terms 
+        if len(t) > 1 and not all(token.is_stop for token in nlp(t))
+    ]
+    
+    # Remove redundant subphrases
+    final_terms = []
+    for term in sorted(cleaned, key=len, reverse=True):
+        if not any(term in other for other in final_terms):
+            final_terms.append(term)
+    
+    return final_terms
+
+###############
 
 def load_classifier():
     """Load zero-shot classification model"""
@@ -146,10 +208,24 @@ async def startup_event():
     load_classifier()
     load_ner_model()
 
+    try:
+        nltk.download('stopwords')
+        nltk.download('punkt_tab')
+    except Exception as e:
+        logger.error(f"NLTK download error: {e}")
+
 @app.post("/chat", response_model=ChatResponse)
 async def chat_endpoint(request: ChatRequest):
     """Enhanced chat endpoint with dynamic model selection"""
+
     try:
+        spacy_search = extract_search_terms(request.prompt.lower())
+        logger.info(f"Extracted search terms using Spacy: {spacy_search}")
+    except Exception as e:
+        logger.error(f"Error extracting search terms using Spacy: {e}")
+        
+    try:
+
         search_results = []
         search_context = ""
 

requirements.txt

@@ -7,4 +7,8 @@ accelerate==0.24.0
 duckduckgo-search
 requests==2.31.0
 python-multipart==0.0.6
-keras
+keras
+rake_nltk
+nltk
+spacy 
+en_core_web_sm @ https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl