Modified scrapper setings

code/web_scraper_allegro.py

@@ -29,9 +29,14 @@ def scrape_allegro_offer(url: str):
     """Zwraca dane aukcji bez zapisywania na dysk"""
     options = uc.ChromeOptions()
     options.add_argument("--window-position=-3000,0")
-    options.add_argument("--headless")
+    options.add_argument("--headless=new")
     options.add_argument("--no-sandbox")
     options.add_argument("--disable-dev-shm-usage")
+    options.add_argument("--disable-blink-features=AutomationControlled")
+    options.add_argument("--start-maximized")
+    options.add_argument("--disable-extensions")
+    # set a realistic user agent
+    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36")
     
     # Ustawienie binarki Chrome'a
     if os.path.exists('/usr/bin/google-chrome'):
@@ -46,7 +51,16 @@ def scrape_allegro_offer(url: str):
     try:
         print(f"🔍 Allegro: {url}")
         driver.get(url)
-        time.sleep(10)
+        # Wait for dynamic content (images / title) to load
+        from selenium.webdriver.support.ui import WebDriverWait
+        from selenium.webdriver.support import expected_conditions as EC
+        try:
+            WebDriverWait(driver, 15).until(
+                EC.presence_of_element_located((By.TAG_NAME, "img"))
+            )
+        except Exception:
+            # fallback short sleep
+            time.sleep(5)
         
         # TITLE
         try:
@@ -71,28 +85,59 @@ def scrape_allegro_offer(url: str):
         
         # DESCRIPTION
         try:
-            description_element = driver.find_element(By.CSS_SELECTOR, "div._0d3bd_am0a-")
-            description_content = description_element.text
+            # try a few common selectors
+            description_element = None
+            for sel in ["div._0d3bd_am0a-", "div[itemprop='description']", "div#description"]:
+                try:
+                    description_element = driver.find_element(By.CSS_SELECTOR, sel)
+                    if description_element:
+                        break
+                except:
+                    continue
+            description_content = description_element.text if description_element else "No description"
         except:
             description_content = "No description"
         
         # IMAGES
         unique_links = set()
         try:
-            images = driver.find_elements(By.CSS_SELECTOR, ".msub_80.m9tr_5r._07951_IOf8s")
+            # more resilient image selector: look for any img with allegro domains
+            imgs = driver.find_elements(By.XPATH, "//img[contains(@src,'allegroimg.com') or contains(@src,'allegrostatic')]")
+            # also try thumbnails / data-src
+            if not imgs:
+                imgs = driver.find_elements(By.TAG_NAME, "img")
+
             allowed_sizes = ["/s128/", "/s360/", "/s512/", "/s720/", "/s1024/", "/s1440/", "/original/"]
-            for img in images:
-                src = img.get_attribute("src")
-                if src and "allegroimg.com" in src:
-                    if not any(size in src for size in allowed_sizes):
-                        continue
-                    for size in allowed_sizes:
-                        src = src.replace(size, "/original/")
-                    unique_links.add(src)
+            for img in imgs:
+                try:
+                    src = img.get_attribute("src") or img.get_attribute("data-src") or img.get_attribute("data-lazy-src")
+                    if not src:
+                        # try srcset
+                        srcset = img.get_attribute("srcset")
+                        if srcset:
+                            src = srcset.split()[0]
+                    if src and ("allegroimg.com" in src or "allegrostatic" in src):
+                        # normalize to original size when possible
+                        for size in allowed_sizes:
+                            if size in src:
+                                src = src.replace(size, "/original/")
+                                break
+                        unique_links.add(src)
+                except Exception:
+                    continue
         except Exception as e:
             print(f"Image error: {e}")
+
+        # If site served a captcha / anti-bot page, include a preview for debugging
+        blocked_preview = None
+        try:
+            page_src = driver.page_source
+            if "captcha-delivery" in page_src or "Please enable JS" in page_src or "ad blocker" in page_src.lower():
+                blocked_preview = page_src[:2000]
+        except Exception:
+            blocked_preview = None
         
-        return {
+        result = {
             "platform": "allegro",
             "url": url,
             "title": title_str,
@@ -100,6 +145,9 @@ def scrape_allegro_offer(url: str):
             "parameters": parameter_list,
             "image_urls": list(unique_links)
         }
+        if blocked_preview:
+            result["blocked_preview"] = blocked_preview
+        return result
     
     finally:
         driver.quit()