Update extract.py

extract.py

@@ -1,8 +1,13 @@
 from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.chrome.service import Service
 from selenium.common.exceptions import WebDriverException
+from bs4 import BeautifulSoup
 import time
 import random
-from bs4 import BeautifulSoup
+import requests
+from fp.fp import FreeProxy
 
 # Lista de User Agents para rotar
 user_agents = [
@@ -28,6 +33,7 @@ user_agents = [
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:91.0) Gecko/20100101 Firefox/91.0",
 ]
 
+
 def get_random_user_agent():
     return random.choice(user_agents)
 
@@ -37,12 +43,20 @@ def get_random_window_size():
     ]
     return random.choice(window_sizes)
 
+def get_proxy():
+    proxy = FreeProxy(rand=True, timeout=1).get()
+    return proxy
+
 def extract_data(user_input, mode):
+    proxy = get_proxy()
+    proxy_url = f"http://{proxy}"
+    
     options = webdriver.ChromeOptions()
     options.add_argument('--headless')
     options.add_argument('--no-sandbox')
     options.add_argument('--disable-dev-shm-usage')
     options.add_argument(f"user-agent={get_random_user_agent()}")
+    options.add_argument('--proxy-server=%s' % proxy_url)
 
     try:
         wd = webdriver.Chrome(options=options)
@@ -60,6 +74,7 @@ def extract_data(user_input, mode):
         page_content = wd.page_source
 
     except WebDriverException as e:
+        print(f"Request failed with proxy {proxy_url}. Error: {e}")
         return []
     finally:
         if wd: