Use spacy for summarizer preprocessing

app.py

@@ -58,9 +58,6 @@ summarization_pipeline = None
 nlp = spacy.load("en_core_web_sm")
 rake = Rake()
 
-
-############# SPACY TEST
-
 def extract_search_terms(text: str) -> List[str]:
     """Extract enhanced search terms using combined NER, syntax, and keywords"""
     doc = nlp(text)
@@ -111,8 +108,6 @@ def clean_terms(terms: List[str]) -> List[str]:
     
     return final_terms
 
-###############
-
 def load_classifier():
     """Load zero-shot classification model"""
     global classifier_pipeline
@@ -184,11 +179,20 @@ def search_web(query: str, max_results: int = 5) -> List[Dict[str, Any]]:
 
 def format_search_context(results: List[Dict[str, Any]]) -> str:
     """Create condensed context from search results"""
-    return "\n".join(
+    return  "\n".join(
         f"{i+1}. {res['title']}: {res['body'][:200]}"
         for i, res in enumerate(results[:5])
     )
 
+def preprocess_text(text: str) -> str:
+    """Use spaCy for fast text cleaning/normalization"""
+    doc = nlp(text)
+    # Lemmatize and remove stopwords
+    return " ".join([
+        token.lemma_ for token in doc
+        if not token.is_stop and not token.is_punct
+    ])[:1024]
+    
 @app.on_event("startup")
 async def startup_event():
     """Initialize core models on startup"""
@@ -197,7 +201,7 @@ async def startup_event():
 @app.post("/chat", response_model=ChatResponse)
 async def chat_endpoint(request: ChatRequest):
     """Enhanced chat endpoint with dynamic model selection"""
-      
+    logger.info(f"Request: {request.prompt}")  
     try:
         search_results = []
         search_context = ""
@@ -229,18 +233,18 @@ async def chat_endpoint(request: ChatRequest):
             if not summarization_pipeline:
                 load_summarization_model()
             # Handle long contexts safely
-            inputs = summarization_pipeline.tokenizer(
-                search_context or request.prompt,
-                truncation=True,
-                max_length=1024,
-                return_tensors="pt"
-            )
-            processed_context = summarization_pipeline.tokenizer.decode(
-                inputs['input_ids'][0],
-                skip_special_tokens=True
-            )
+            # inputs = summarization_pipeline.tokenizer(
+            #     search_context or request.prompt,
+            #     truncation=True,
+            #     max_length=1024,
+            #     return_tensors="pt"
+            # )
+            # processed_context = summarization_pipeline.tokenizer.decode(
+            #     inputs['input_ids'][0],
+            #     skip_special_tokens=True
+            # )
             response = summarization_pipeline(
-                processed_context,
+                preprocess_text(search_context.lower() or request.prompt.lower()),
                 max_length=150,
                 min_length=30,
                 do_sample=False