replaced chromedriver/selenium with zenrows

app.py

@@ -110,8 +110,8 @@ if __name__ == '__main__':
     #         libnss3=2:3.26.2-1.1+deb9u1 \
     #         libgconf-2-4=3.2.6-4+b1 \
     #         libfontconfig1=2.11.0-6.7+b1
-    check_call(['apt-get', 'install', '-y', 'libglib2.0-0 libnss3 libgconf-2-4 libfontconfig1'],
-               stdout=open(os.devnull, 'wb'), stderr=STDOUT)
+    # check_call(['apt-get', 'install', '-y', 'libglib2.0-0 libnss3 libgconf-2-4 libfontconfig1'],
+    #            stdout=open(os.devnull, 'wb'), stderr=STDOUT)
 
     logger.info("Launching Gradio ChatInterface for searchbot...")
 

requirements.txt

@@ -8,8 +8,9 @@ llama-index-embeddings-openai
 llama-index-llms-openai
 # needed for simpledirectoryreader to work
 llama-index-readers-file
-selenium==4.22.0
+# selenium==4.22.0
 unstructured
 requests
-chromium
+# chromium
+zenrows
 

web_search.py

@@ -2,7 +2,6 @@ import copy
 import json
 import logging
 import os
-import stat
 import time
 import traceback
 import urllib.parse as en
@@ -10,11 +9,8 @@ import warnings
 from itertools import zip_longest
 
 import requests
-import selenium.common.exceptions
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.chrome.service import Service as ChromeService
 from unstructured.partition.html import partition_html
+from zenrows import ZenRowsClient
 
 from llmsearch import site_stats
 # this import style works in pycharm
@@ -27,6 +23,7 @@ from llmsearch import utilityV2 as ut
 # from llmsearch import utilityV2 as ut
 
 logger = logging.getLogger("agent_logger")
+logger = logging.getLogger("agent_logger")
 
 
 # todo drop blocked pages > see og llmsearch code
@@ -68,39 +65,45 @@ def process_url(url, timeout):
     try:
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
-            options = Options()
-            options.page_load_strategy = "eager"
-            options.add_argument("--headless")
-            options.add_argument("--no-sandbox")
-            options.add_argument("--disable-dev-shm-usage")
-
-            options.add_argument("start-maximized")
-            options.add_argument("disable-infobars")
-            options.add_argument("--disable-extensions")
-            options.add_argument("--disable-gpu")
-            options.add_argument("--disable-dev-shm-usage")
+            # options = Options()
+            # options.page_load_strategy = "eager"
+            # options.add_argument("--headless")
+            # options.add_argument("--no-sandbox")
+            # options.add_argument("--disable-dev-shm-usage")
+            #
+            # options.add_argument("start-maximized")
+            # options.add_argument("disable-infobars")
+            # options.add_argument("--disable-extensions")
+            # options.add_argument("--disable-gpu")
+            # options.add_argument("--disable-dev-shm-usage")
             result = ""
             # make driver exec
-            os.chmod('chromedriver-linux64/chromedriver', stat.S_IEXEC)
+            # os.chmod('chromedriver-linux64/chromedriver', stat.S_IEXEC)
             try:
-                driver = webdriver.Chrome(service=ChromeService(executable_path='chromedriver-linux64/chromedriver'),
-                                          options=options)
-                logger.info(f"*****setting page load timeout {timeout}")
-                driver.set_page_load_timeout(timeout)
-                driver.get(url)
-                response = driver.page_source
-                result = response_text_extract(url=url, response=response)
-            except selenium.common.exceptions.TimeoutException:
-                return "", url
-            except selenium.common.exceptions.WebDriverException:
+                # driver = webdriver.Chrome(service=ChromeService(executable_path='chromedriver-linux64/chromedriver'),
+                #                           options=options)
+                # logger.info(f"*****setting page load timeout {timeout}")
+                # driver.set_page_load_timeout(timeout)
+                # driver.get(url)
+                # response = driver.page_source
+                client = ZenRowsClient(os.getenv('zenrows_api_key'))
+                response = client.get(url)
+                # result = response_text_extract(url=url, response=response)
+                result = response.text
+            except Exception:
                 traceback.print_exc()
-                logger.info(f"webdriver failed to load")
                 return "", url
+            # except selenium.common.exceptions.TimeoutException:
+            #     return "", url
+            # except selenium.common.exceptions.WebDriverException:
+            #     traceback.print_exc()
+            #     logger.info(f"webdriver failed to load")
+            #     return "", url
     except Exception:
         traceback.print_exc()
         logger.info(f"{site} err")
         pass
-    logger.info(f"Processed {site}: {len(response)} / {len(result)} {int((time.time() - start_time) * 1000)} ms")
+    logger.info(f"Processed {site}: {len(response.text)} / {len(result)} {int((time.time() - start_time) * 1000)} ms")
     return result, url