Update app.py

app.py

@@ -1097,26 +1097,29 @@ def search_knowledge_base(query):
     # Retrieve the top 5 most relevant documents
     retrieved_docs = st.session_state["faiss_db"].similarity_search(query, k=3)
     return retrieved_docs
-    
-async def validate_url(session, url):
+# Asynchronous function to validate and verify link content
+async def validate_and_fetch_content(session, url, query):
     try:
-        async with session.head(url, allow_redirects=True, timeout=5) as response:
+        async with session.get(url, allow_redirects=True, timeout=10) as response:
             if response.status == 200:
-                return True
-            return False
+                content = await response.text()
+                if query.lower() in content.lower():
+                    return {"url": url, "valid": True, "contains_query": True}
+            return {"url": url, "valid": True, "contains_query": False}
     except Exception:
-        return False
+        return {"url": url, "valid": False, "contains_query": False}
 
 # Function to validate a batch of URLs asynchronously
-async def validate_links_async(urls):
+async def validate_links_with_content(urls, query):
     async with aiohttp.ClientSession() as session:
-        tasks = [validate_url(session, url) for url in urls]
+        tasks = [validate_and_fetch_content(session, url, query) for url in urls]
         results = await asyncio.gather(*tasks)
         return results
 
 # Function to perform a Google search and validate links
 def google_search(query):
     try:
+        # Step 1: Perform the search using SERP API
         search_client = Client(api_key=serper_api_key)
         results = search_client.search({"engine": "google", "q": query})
         
@@ -1125,22 +1128,20 @@ def google_search(query):
         urls = [result.get("link") for result in organic_results if result.get("link")]
         snippets = [result.get("snippet") for result in organic_results if result.get("link")]
 
-        # Validate URLs asynchronously
-        valid_links = asyncio.run(validate_links_async(urls))
-        
-        # Filter valid snippets and links
-        valid_results = [
-            {"snippet": snippet, "link": url} 
-            for snippet, url, is_valid in zip(snippets, urls, valid_links) 
-            if is_valid
-        ]
-        
+        # Step 2: Validate and verify link content
+        validated_links = asyncio.run(validate_links_with_content(urls, query))
+
+        # Step 3: Combine valid snippets and URLs with verified content
+        valid_results = []
+        for snippet, validation in zip(snippets, validated_links):
+            if validation["valid"] and validation["contains_query"]:
+                valid_results.append({"snippet": snippet, "link": validation["url"]})
+
         return valid_results
     
     except Exception as e:
         logger.error(f"Error in Google search: {e}")
         return [{"snippet": "Error occurred during Google search", "link": ""}]
-
 def rag_response(query, selected_doc_ids=None):
     """
     Handle queries by searching both the main knowledge base and the selected documents.