Update app.py

app.py

@@ -388,7 +388,7 @@ class RAGIndex:
         return answer
 
     def answer(self, question: str) -> str:
-        """Answer a question using RAG with improved generation."""
+        """Answer a question using RAG - simplified extractive approach."""
         if not self.initialized:
             return "❌ Assistant not properly initialized. Please check the logs."
     
@@ -402,7 +402,7 @@ class RAGIndex:
                 f"Supported formats: .txt, .md, .pdf, .docx"
             )
     
-        # 1) Retrieve relevant contexts
+        # Retrieve relevant contexts
         contexts = self.retrieve(question, top_k=3)
     
         if not contexts:
@@ -412,64 +412,69 @@ class RAGIndex:
             )
     
         used_sources = set()
+        best_context = None
+        best_score = 0
         
-        # 2) Collect and clean the best contexts
-        evidence_parts = []
+        # Find the best matching context
         for ctx, source, score in contexts:
             used_sources.add(source)
-            cleaned_ctx = clean_context_text(ctx)
-            if cleaned_ctx.strip():
-                evidence_parts.append(cleaned_ctx)
+            if score > best_score:
+                best_score = score
+                best_context = ctx
         
-        if not evidence_parts:
-            return (
-                f"{NO_ANSWER_MSG}\n\n"
-                f"💡 Try rephrasing your question or adding more detailed documents to the knowledge base."
-            )
+        if not best_context:
+            return f"{NO_ANSWER_MSG}"
         
-        # Combine contexts - use MORE context for better answers
-        combined_context = " ".join(evidence_parts[:3])[:1500]  # Top 3, up to 1500 chars
+        # AGGRESSIVE cleaning of the context
+        def deep_clean(text):
+            """Remove ALL markdown, bullets, numbers, emojis, and formatting."""
+            # Remove emojis and special characters
+            text = re.sub(r'[📘📄🟢🟡🟠✓✗❌⚠️💡📚]', '', text)
+            # Remove markdown headers (# ## ###)
+            text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE)
+            # Remove numbered lists (1. 2. 3.)
+            text = re.sub(r'^\s*\d+\.\s+', '', text, flags=re.MULTILINE)
+            # Remove bullet points (- * •)
+            text = re.sub(r'^\s*[-*•]\s+', '', text, flags=re.MULTILINE)
+            # Remove bold/italic (**text** *text*)
+            text = re.sub(r'\*\*([^*]+)\*\*', r'\1', text)
+            text = re.sub(r'\*([^*]+)\*', r'\1', text)
+            # Remove extra colons from labels
+            text = re.sub(r':\s*$', '', text, flags=re.MULTILINE)
+            # Clean multiple spaces
+            text = re.sub(r'\s+', ' ', text)
+            # Remove "Good:" "Bad:" type prefixes
+            text = re.sub(r'^(Good|Bad|Example|Note):\s*', '', text, flags=re.MULTILINE)
+            return text.strip()
         
-        # 3) Better prompt structure for FLAN-T5
-        answer_prompt = f"""Read the context and answer the question in 2-3 complete sentences.
-    
-    Context: {combined_context}
-    
-    Question: {question}
-    
-    Answer in complete sentences:"""
-    
-        try:
-            answer_text = self._generate_from_context(answer_prompt, max_new_tokens=200)
-            answer_text = answer_text.strip()
-            
-            # Clean up common artifacts
-            answer_text = answer_text.replace("**", "").replace("##", "").strip()
+        cleaned = deep_clean(best_context)
+        
+        # Extract just the most relevant sentences (3-4 sentences max)
+        sentences = [s.strip() + '.' for s in cleaned.split('.') if len(s.strip()) > 20]
+        answer_text = ' '.join(sentences[:4])  # First 4 good sentences
+        
+        # If we got good text, try to generate a natural answer
+        if len(answer_text) > 50:
+            # Simple prompt for FLAN-T5
+            prompt = f"Question: {question}\n\nInformation: {answer_text[:800]}\n\nWrite a clear answer in 2-3 sentences:"
             
-            # If answer is poor quality, try alternative approach
-            if (len(answer_text) < 20 or 
-                answer_text.count(":") > 3 or 
-                answer_text.startswith("Do NOT") or
-                "tone tone tone" in answer_text.lower()):
+            try:
+                generated = self._generate_from_context(prompt, max_new_tokens=150)
+                generated = generated.strip()
                 
-                # Try extractive approach: just clean and present the best context
-                best_context = evidence_parts[0]
-                # Remove list markers and clean
-                best_context = re.sub(r'^\s*[-*]\s*', '', best_context, flags=re.MULTILINE)
-                best_context = re.sub(r'^\s*\d+\.\s*', '', best_context, flags=re.MULTILINE)
-                # Take first few sentences
-                sentences = best_context.split('.')[:3]
-                answer_text = '. '.join(s.strip() for s in sentences if s.strip()) + '.'
-            
-        except Exception as e:
-            print(f"Generation error: {e}")
-            return (
-                "There was an error while generating the answer. "
-                "Please try again with a shorter question or different wording."
-            )
-    
+                # Only use generated answer if it looks good
+                if (len(generated) > 30 and 
+                    not generated.startswith(("Do NOT", "You are", "##", "**")) and
+                    generated.count(':') < 3):
+                    answer_text = generated
+                # Otherwise, keep the cleaned extractive answer
+                    
+            except Exception as e:
+                print(f"Generation error (using extractive fallback): {e}")
+                # Keep the cleaned extractive answer
+        
         sources_str = ", ".join(sorted(used_sources)) if used_sources else "N/A"
-    
+        
         return (
             f"**Answer:** {answer_text}\n\n"
             f"**Sources:** {sources_str}"