Fix Amazon crawling: Add Start URL Retry logic and more block keywords

src/crawler.py

@@ -95,7 +95,10 @@ class Crawler:
             "captcha",
             "security check",
             "turn on javascript",
-            "challenge.js"
+            "challenge.js",
+            "api-services-support@amazon.com",
+            "we just need to make sure you're not a robot",
+            "type the characters you see in this image"
         ]
         
         if any(k in lower_content for k in block_keywords):
@@ -268,6 +271,18 @@ class Crawler:
                     # Unpack result
                     _, images, raw_links = result
                     
+                    # --- CRITICAL FIX: Start URL Retry for Dynamic/Blocked Pages ---
+                    if url == start_url and len(raw_links) < 5:
+                        logger.warning(f"Start URL {url} returned only {len(raw_links)} links. Likely JS-heavy or blocked. Forcing Playwright retry...")
+                        # Run Playwright directly in main thread for the seed
+                        pw_content = self._fetch_playwright(url)
+                        if pw_content:
+                            images = self.extract_images(pw_content, url)
+                            soup = BeautifulSoup(pw_content, 'html.parser')
+                            raw_links = [link.get('href') for link in soup.find_all('a', href=True)]
+                            logger.info(f"Playwright retry found {len(raw_links)} links.")
+                    # ---------------------------------------------------------------
+
                     logger.info(f"Crawled [{pages_crawled + 1}]: {url}")
                     site_data[url] = images
                     pages_crawled += 1