Optimize memory usage to prevent crashes

Dockerfile

@@ -31,4 +31,4 @@ EXPOSE 7860
 
 # Command to run the application using Gunicorn
 # Bind to 0.0.0.0:7860 as required by HF Spaces
-CMD ["gunicorn", "-b", "0.0.0.0:7860", "--timeout", "120", "--workers", "2", "api:app"]
+CMD ["gunicorn", "-b", "0.0.0.0:7860", "--timeout", "120", "--workers", "1", "--threads", "2", "api:app"]

src/crawler.py

@@ -149,6 +149,7 @@ class Crawler:
                 )
                 
                 page = context.new_page()
+                try:
                 try:
                     logger.info(f"Playwright navigating to {url}...")
                     page.goto(url, timeout=60000, wait_until="domcontentloaded")
@@ -160,30 +161,24 @@ class Crawler:
                     title = page.title()
                     logger.info(f"Page Title: {title}")
                     
-                    # CAPTCHA Detection via Selectors
-                    # Amazon often uses #captchacharacters
-                    if page.locator("input#captchacharacters").count() > 0 or "Robot Check" in title:
-                        logger.error("Playwright hit visible CAPTCHA. Manual intervention required.")
-                        self.blocked_reason = "Hard Catcha/Robot Check"
-                        
-                        # Save screenshot for proof
-                        page.screenshot(path="amazon.png")
-                        logger.info("Saved screenshot to amazon.png")
-                        
-                        # Save HTML
-                        with open("debug_playwright.html", "w", encoding="utf-8") as f:
-                            f.write(page.content())
-                        
-                        return None # Return None to signify blocked/useless content
-
                     content = page.content()
+                    
+                    # Explicit cleanup
+                    page.close()
+                    context.close()
+                    browser.close()
+                    
                     return content
                     
                 except Exception as nav:
                      logger.error(f"Playwright navigation error: {nav}")
                      return None
                 finally:
-                    browser.close()
+                    # Ensure browser is closed even if errors occur
+                    try:
+                        browser.close()
+                    except:
+                        pass
 
         except Exception as e:
             logger.error(f"Playwright critical error: {e}")