Update reasoning/scraper.py

reasoning/scraper.py

@@ -8,11 +8,30 @@ HEADERS = {
     "User-Agent": "Mozilla/5.0 (MVI-AI Knowledge Engine)"
 }
 
-# SAFE SOURCE ONLY
 SOURCES = {
     "wikipedia": "https://en.wikipedia.org/wiki/{query}"
 }
 
+# -------------------------
+# QUERY NORMALIZATION
+# -------------------------
+
+def normalize_query(query: str) -> str:
+    query = query.lower()
+
+    stop_phrases = [
+        "what is", "who is", "define",
+        "explain", "tell me about",
+        "what are", "how does"
+    ]
+
+    for phrase in stop_phrases:
+        query = query.replace(phrase, "")
+
+    query = re.sub(r'[^\w\s]', '', query)
+
+    return query.strip()
+
 
 # -------------------------
 # CLEANING FUNCTIONS
@@ -33,6 +52,7 @@ def is_blocked(text: str) -> bool:
         "access denied",
         "enable javascript"
     ]
+
     text_lower = text.lower()
     return any(b in text_lower for b in blockers)
 
@@ -67,7 +87,6 @@ def scrape_page(url: str) -> str:
 
         soup = BeautifulSoup(r.text, "html.parser")
 
-        # Remove junk tags
         for tag in soup(["script", "style", "noscript"]):
             tag.decompose()
 
@@ -89,9 +108,14 @@ def scrape_page(url: str) -> str:
 def scrape_knowledge(query: str, limit: int = 20) -> List[Dict]:
     knowledge = []
 
+    clean_query = normalize_query(query)
+
+    if not clean_query:
+        return knowledge
+
     for source_name, url in SOURCES.items():
         try:
-            formatted_query = query.strip().replace(" ", "_")
+            formatted_query = clean_query.replace(" ", "_")
             full_url = url.format(query=formatted_query)
 
             page_text = scrape_page(full_url)
@@ -113,4 +137,4 @@ def scrape_knowledge(query: str, limit: int = 20) -> List[Dict]:
         except Exception:
             continue
 
-    return knowledge
+    return knowledge