links

conversation.py

@@ -4,6 +4,8 @@ import gradio as gr
 import requests
 from bs4 import BeautifulSoup
 import urllib.parse
+from selenium import webdriver
+from webdriver_manager.chrome import ChromeDriverManager
 
 try:
     from dotenv import load_dotenv
@@ -43,9 +45,36 @@ class Conversation:
         # ... your existing get_data implementation ...
         # Replace `messages` with `self.messages`
 
+        def extract_html_content(url):
+            response = requests.get(url)
+            return response.text
+
+        def extract_js_content(url):
+            options = webdriver.ChromeOptions()
+            options.add_argument('--headless')
+            driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
+            driver.get(url)
+            rendered_content = driver.page_source
+            driver.quit()
+            return rendered_content
+
+        def smart_scraper(url):
+            html_content = extract_html_content(url)
+            selector_to_find = "body"
+            
+            # Check if the content is incomplete or if a specific tag is missing
+            # if not html_content or not html_content.find(selector_to_find):
+            if not html_content or not html_content.find(selector_to_find):
+                # If incomplete, use Selenium to render JavaScript
+                print("Using Selenium for JavaScript rendering...")
+                js_content = extract_js_content(url)
+                return js_content
+            else:
+                return html_content
+
         url = self.to_valid_url(old_url)
         self.messages
-        html = requests.get(url).text
+        html = smart_scraper(url)
         doc = BeautifulSoup(html, 'html.parser')
         if not doc:
           raise ValueError("Please try again")
@@ -53,6 +82,7 @@ class Conversation:
         headings_1 = [e.text for e in doc.find_all('h1')]
         headings_2 = [e.text for e in doc.find_all('h2')]
         # headings_3 = [e.text for e in doc.find_all('h3')]
+        links = [e.text for e in doc.find_all('a')]
         paragraphs = [e.text for e in doc.find_all('p')]
         # spans = [e.text for e in doc.find_all('span')]
         joined_paragraphs = (' '.join(paragraphs))
@@ -64,6 +94,7 @@ class Conversation:
         self.messages.append({'role': 'system', 'content': "You are a helpful assistant that must answer questions about a website."})
         self.messages.append({'role': 'system', 'content': f"here are the h1s - {headings_1}"})
         self.messages.append({'role': 'system', 'content': f"here are the h2s - {headings_2}"})
+        self.messages.append({'role': 'system', 'content': f"here are the links - {links}"})
         # messages.append({'role': 'system', 'content': f"here are the h3s - {headings_3}"})
         self.messages.append({'role': 'system', 'content': f"here are the paragraphs - {paragraphs}"})
         # messages.append({'role': 'system', 'content': f"here are the spans - {spans}"})