Update chatbot_rag.py

chatbot_rag.py

@@ -88,25 +88,32 @@ def build_qa():
 
 
     def hf_to_str(x):
-        """Convert Hugging Face pipeline output to plain string (clean + generalized)."""
+        """Convert Hugging Face pipeline output to clean plain text."""
         if isinstance(x, list) and "generated_text" in x[0]:
             text = x[0]["generated_text"]
         else:
             text = str(x)
     
-        # 1. Remove markdown/code artifacts
-        text = text.replace("```", "").replace("#", "").strip()
+        # 1. Remove code-like artifacts
+        text = re.sub(r"def\s+\w+\(.*?\):.*", "", text, flags=re.DOTALL)
+        text = re.sub(r"(print\(.*?\))", "", text)
+        text = re.sub(r"text\s*\+=.*", "", text)
     
-        # 2. Normalize whitespace & line breaks
-        text = re.sub(r"\s+", " ", text)
+        # 2. Remove markdown/code fences
+        text = text.replace("```", "").replace("'''", "").replace('"""', "")
     
-        # 3. Remove duplicated consecutive phrases (up to ~5 words repeated)
-        text = re.sub(r"\b(\w+\s+){1,5}(\1){2,}", r"\1", text)
+        # 3. Normalize whitespace & line breaks
+        text = re.sub(r"\s+", " ", text)
     
-        # 4. Trim leading/trailing junk
-        text = text.strip(" .,-\n\t")
+        # 4. Remove repeated sentences (simple dedupe)
+        sentences = []
+        for s in re.split(r"(?<=[.!?])\s+", text):
+            if s not in sentences:
+                sentences.append(s)
+        text = " ".join(sentences)
     
-        return text
+        # 5. Trim
+        return text.strip()
 
 
     # 7. RAG chain