Update extract.py

extract.py

@@ -6,9 +6,12 @@ from selenium.common.exceptions import WebDriverException
 from bs4 import BeautifulSoup
 import time
 import random
-import requests
+import logging
 from fp.fp import FreeProxy
 
+# Configurar el logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
 # Lista de User Agents para rotar
 user_agents = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
@@ -18,22 +21,8 @@ user_agents = [
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",
     "Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:87.0) Gecko/20100101 Firefox/87.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:88.0) Gecko/20100101 Firefox/88.0",
-    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0",
-    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:91.0) Gecko/20100101 Firefox/91.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:91.0) Gecko/20100101 Firefox/91.0",
 ]
 
-
 def get_random_user_agent():
     return random.choice(user_agents)
 
@@ -58,6 +47,7 @@ def extract_data(user_input, mode):
     options.add_argument(f"user-agent={get_random_user_agent()}")
     options.add_argument('--proxy-server=%s' % proxy_url)
 
+    wd = None
     try:
         wd = webdriver.Chrome(options=options)
         window_size = get_random_window_size()
@@ -65,6 +55,9 @@ def extract_data(user_input, mode):
         
         # Construir la URL de búsqueda
         url_busqueda = f"https://app.neilpatel.com/es/traffic_analyzer/keywords?domain={user_input}&lang=es&locId=2724&mode={mode}"
+        
+        logging.info(f"Making request to {url_busqueda} with IP: {proxy_url}")
+        
         wd.get(url_busqueda)
 
         # Espera aleatoria para simular el comportamiento humano
@@ -73,8 +66,15 @@ def extract_data(user_input, mode):
         # Obtener el contenido de la página
         page_content = wd.page_source
 
+        # Obtener el código de respuesta HTTP
+        response_status = wd.execute_script("return document.readyState")
+        if response_status == "complete":
+            logging.info(f"Request with IP: {proxy_url} returned status code 200")
+        else:
+            logging.warning(f"Request with IP: {proxy_url} did not return status code 200")
+
     except WebDriverException as e:
-        print(f"Request failed with proxy {proxy_url}. Error: {e}")
+        logging.error(f"Request failed with proxy {proxy_url}. Error: {e}")
         return []
     finally:
         if wd:
@@ -132,4 +132,4 @@ def extract_data(user_input, mode):
 
     # Parsear el texto filtrado
     datos_parseados = parsear_texto(lineas_filtradas)
-    return datos_parseados
+    return datos_parseados