Update extract.py

extract.py

@@ -1,6 +1,5 @@
 from selenium import webdriver
 from selenium.webdriver.common.by import By
-from selenium.webdriver.chrome.service import Service
 from selenium.common.exceptions import WebDriverException
 from bs4 import BeautifulSoup
 import time
@@ -9,14 +8,14 @@ import random
 # Genera un User Agent aleatorio
 def generate_random_user_agent():
     browsers = {
-        "Chrome": range(70, 115),  # Versiones 70 a 114
-        "Firefox": range(60, 110),  # Versiones 60 a 109
+        "Chrome": range(70, 115),
+        "Firefox": range(60, 110),
         "Safari": ["13.1", "14.0", "15.0", "16.0", "17.0"],
-        "Edge": range(80, 105),  # Versiones 80 a 104
-        "Opera": range(50, 90),  # Versiones 50 a 89
-        "Brave": range(1, 40),  # Versiones 1 a 39
-        "Vivaldi": range(2, 6),  # Versiones 2 a 5
-        "UC Browser": [f"13.{v}" for v in range(0, 21)]  # Versiones 13.0 a 13.20
+        "Edge": range(80, 105),
+        "Opera": range(50, 90),
+        "Brave": range(1, 40),
+        "Vivaldi": range(2, 6),
+        "UC Browser": [f"13.{v}" for v in range(0, 21)]
     }
 
     operating_systems = [
@@ -36,6 +35,29 @@ def generate_random_user_agent():
 
     return f"Mozilla/5.0 ({os_version}; {architecture}; {language}) AppleWebKit/537.36 (KHTML, like Gecko) {browser}/{version} Safari/537.36"
 
+# Generar un tamaño de ventana aleatorio
+def get_random_window_size():
+    window_sizes = [
+        (1920, 1080), (1366, 768), (1440, 900), (1536, 864), (1280, 800), (1280, 720), (1024, 768)
+    ]
+    return random.choice(window_sizes)
+
+# Generar cabeceras HTTP aleatorias
+def generate_random_headers():
+    languages = ["es-ES,es;q=0.9", "en-US,en;q=0.9", "fr-FR,fr;q=0.9", "de-DE,de;q=0.9"]
+    accept = ["text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 
+              "application/json,text/html;q=0.9"]
+    return {
+        'accept-language': random.choice(languages),
+        'accept': random.choice(accept)
+    }
+
+# Simular scroll en la página
+def simulate_scroll(driver):
+    scroll_pause_time = random.uniform(1, 3)
+    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+    time.sleep(scroll_pause_time)
+
 def extract_data(user_input, mode):
     options = webdriver.ChromeOptions()
     options.add_argument('--headless')
@@ -45,17 +67,27 @@ def extract_data(user_input, mode):
     # Usa un User Agent aleatorio para cada petición
     options.add_argument(f"user-agent={generate_random_user_agent()}")
 
+    # Generar cabeceras HTTP aleatorias
+    headers = generate_random_headers()
+    options.add_argument(f"accept-language={headers['accept-language']}")
+    options.add_argument(f"accept={headers['accept']}")
+
+    wd = None
     try:
         wd = webdriver.Chrome(options=options)
-        wd.set_window_size(1080, 720)
+        window_size = get_random_window_size()
+        wd.set_window_size(window_size[0], window_size[1])  # Ajusta el tamaño de la ventana aleatoriamente
         
         # Construir la URL de búsqueda
         url_busqueda = f"https://app.neilpatel.com/es/traffic_analyzer/keywords?domain={user_input}&lang=es&locId=2724&mode={mode}"
         wd.get(url_busqueda)
 
+        # Simular scroll en la página
+        simulate_scroll(wd)
+
         # Espera aleatoria para simular el comportamiento humano
-        time.sleep(random.uniform(10, 20))
-        
+        time.sleep(random.uniform(3, 5))  # Espera más larga para simular un comportamiento humano
+
         # Obtener el contenido de la página
         page_content = wd.page_source