feat(browsers): Add option to retry tabs that gives errors

scrapling/engines/_browsers/_base.py

@@ -310,6 +310,14 @@ class BaseSessionMixin:
             if config.additional_args:
                 self._context_options.update(config.additional_args)
 
+    @staticmethod
+    def _is_retriable(error: Exception) -> bool:
+        """Check if an error is retriable (transient network/timeout issues)."""
+        if isinstance(error, TimeoutError):
+            return True
+        error_msg = str(error).lower()
+        return "net::" in error_msg or "failed to get response" in error_msg
+
 
 class DynamicSessionMixin(BaseSessionMixin):
     def __validate__(self, **params):

scrapling/engines/_browsers/_controllers.py

@@ -1,3 +1,6 @@
+from time import sleep as time_sleep
+from asyncio import sleep as asyncio_sleep
+
 from playwright.sync_api import (
     Locator,
     Playwright,
@@ -115,52 +118,56 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
             else None
         )
 
-        page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        final_response = [None]
-        handle_response = self._create_response_handler(page_info, final_response)
-
-        try:  # pragma: no cover
-            # Navigate to URL and wait for a specified state
-            page_info.page.on("response", handle_response)
-            first_response = page_info.page.goto(url, referer=referer)
-            self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-
-            if not first_response:
-                raise RuntimeError(f"Failed to get response for {url}")
-
-            if params.page_action:
-                try:
-                    _ = params.page_action(page_info.page)
-                except Exception as e:  # pragma: no cover
-                    log.error(f"Error executing page_action: {e}")
-
-            if params.wait_selector:
-                try:
-                    waiter: Locator = page_info.page.locator(params.wait_selector)
-                    waiter.first.wait_for(state=params.wait_selector_state)
-                    # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                    self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                except Exception as e:  # pragma: no cover
-                    log.error(f"Error waiting for selector {params.wait_selector}: {e}")
-
-            page_info.page.wait_for_timeout(params.wait)
+        for attempt in range(self._config.retries):
+            page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
+            final_response = [None]
+            handle_response = self._create_response_handler(page_info, final_response)
+
+            try:  # pragma: no cover
+                page_info.page.on("response", handle_response)
+                first_response = page_info.page.goto(url, referer=referer)
+                self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+
+                if not first_response:
+                    raise RuntimeError(f"Failed to get response for {url}")
+
+                if params.page_action:
+                    try:
+                        _ = params.page_action(page_info.page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_action: {e}")
+
+                if params.wait_selector:
+                    try:
+                        waiter: Locator = page_info.page.locator(params.wait_selector)
+                        waiter.first.wait_for(state=params.wait_selector_state)
+                        self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error waiting for selector {params.wait_selector}: {e}")
+
+                page_info.page.wait_for_timeout(params.wait)
+
+                response = ResponseFactory.from_playwright_response(
+                    page_info.page, first_response, final_response[0], params.selector_config
+                )
 
-            # Create response object
-            response = ResponseFactory.from_playwright_response(
-                page_info.page, first_response, final_response[0], params.selector_config
-            )
+                page_info.page.close()
+                self.page_pool.pages.remove(page_info)
+                return response
 
-            # Close the page to free up resources
-            page_info.page.close()
-            self.page_pool.pages.remove(page_info)
+            except Exception as e:
+                page_info.mark_error()
+                page_info.page.close()
+                self.page_pool.pages.remove(page_info)
 
-            return response
+                if attempt < self._config.retries - 1 and self._is_retriable(e):
+                    log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
+                    time_sleep(self._config.retry_delay)
+                else:
+                    raise
 
-        except Exception as e:
-            page_info.mark_error()
-            page_info.page.close()
-            self.page_pool.pages.remove(page_info)
-            raise e
+        # For type checking purposes only
+        raise AssertionError("Unreachable: retry loop must return or raise")  # pragma: no cover
 
 
 class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
@@ -252,54 +259,59 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
             else None
         )
 
-        page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        final_response = [None]
-        handle_response = self._create_response_handler(page_info, final_response)
-
-        if TYPE_CHECKING:
-            from playwright.async_api import Page as async_Page
-
-            if not isinstance(page_info.page, async_Page):
-                raise TypeError
-
-        try:
-            # Navigate to URL and wait for a specified state
-            page_info.page.on("response", handle_response)
-            first_response = await page_info.page.goto(url, referer=referer)
-            await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-
-            if not first_response:
-                raise RuntimeError(f"Failed to get response for {url}")
-
-            if params.page_action:
-                try:
-                    _ = await params.page_action(page_info.page)
-                except Exception as e:
-                    log.error(f"Error executing page_action: {e}")
-
-            if params.wait_selector:
-                try:
-                    waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
-                    await waiter.first.wait_for(state=params.wait_selector_state)
-                    # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                    await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                except Exception as e:
-                    log.error(f"Error waiting for selector {params.wait_selector}: {e}")
-
-            await page_info.page.wait_for_timeout(params.wait)
-
-            # Create response object
-            response = await ResponseFactory.from_async_playwright_response(
-                page_info.page, first_response, final_response[0], params.selector_config
-            )
-
-            # Close the page to free up resources
-            await page_info.page.close()
-            self.page_pool.pages.remove(page_info)
-            return response
-
-        except Exception as e:  # pragma: no cover
-            page_info.mark_error()
-            await page_info.page.close()
-            self.page_pool.pages.remove(page_info)
-            raise e
+        for attempt in range(self._config.retries):
+            page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
+            final_response = [None]
+            handle_response = self._create_response_handler(page_info, final_response)
+
+            if TYPE_CHECKING:
+                from playwright.async_api import Page as async_Page
+
+                if not isinstance(page_info.page, async_Page):
+                    raise TypeError
+
+            try:
+                page_info.page.on("response", handle_response)
+                first_response = await page_info.page.goto(url, referer=referer)
+                await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+
+                if not first_response:
+                    raise RuntimeError(f"Failed to get response for {url}")
+
+                if params.page_action:
+                    try:
+                        _ = await params.page_action(page_info.page)
+                    except Exception as e:
+                        log.error(f"Error executing page_action: {e}")
+
+                if params.wait_selector:
+                    try:
+                        waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
+                        await waiter.first.wait_for(state=params.wait_selector_state)
+                        await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+                    except Exception as e:
+                        log.error(f"Error waiting for selector {params.wait_selector}: {e}")
+
+                await page_info.page.wait_for_timeout(params.wait)
+
+                response = await ResponseFactory.from_async_playwright_response(
+                    page_info.page, first_response, final_response[0], params.selector_config
+                )
+
+                await page_info.page.close()
+                self.page_pool.pages.remove(page_info)
+                return response
+
+            except Exception as e:  # pragma: no cover
+                page_info.mark_error()
+                await page_info.page.close()
+                self.page_pool.pages.remove(page_info)
+
+                if attempt < self._config.retries - 1 and self._is_retriable(e):
+                    log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
+                    await asyncio_sleep(self._config.retry_delay)
+                else:
+                    raise
+
+        # For type checking purposes only
+        raise AssertionError("Unreachable: retry loop must return or raise")  # pragma: no cover

scrapling/engines/_browsers/_stealth.py

@@ -1,5 +1,7 @@
 from random import randint
 from re import compile as re_compile
+from time import sleep as time_sleep
+from asyncio import sleep as asyncio_sleep
 
 from playwright.sync_api import (
     Locator,
@@ -202,57 +204,66 @@ class StealthySession(SyncSession, StealthySessionMixin):
             else None
         )
 
-        page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        final_response = [None]
-        handle_response = self._create_response_handler(page_info, final_response)
+        for attempt in range(self._config.retries):
+            page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
+            final_response = [None]
+            handle_response = self._create_response_handler(page_info, final_response)
 
-        try:  # pragma: no cover
-            # Navigate to URL and wait for a specified state
-            page_info.page.on("response", handle_response)
-            first_response = page_info.page.goto(url, referer=referer)
-            self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-
-            if not first_response:
-                raise RuntimeError(f"Failed to get response for {url}")
-
-            if params.solve_cloudflare:
-                self._cloudflare_solver(page_info.page)
-                # Make sure the page is fully loaded after the captcha
+            try:  # pragma: no cover
+                # Navigate to URL and wait for a specified state
+                page_info.page.on("response", handle_response)
+                first_response = page_info.page.goto(url, referer=referer)
                 self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
 
-            if params.page_action:
-                try:
-                    _ = params.page_action(page_info.page)
-                except Exception as e:  # pragma: no cover
-                    log.error(f"Error executing page_action: {e}")
-
-            if params.wait_selector:
-                try:
-                    waiter: Locator = page_info.page.locator(params.wait_selector)
-                    waiter.first.wait_for(state=params.wait_selector_state)
-                    # Wait again after waiting for the selector, helpful with protections like Cloudflare
+                if not first_response:
+                    raise RuntimeError(f"Failed to get response for {url}")
+
+                if params.solve_cloudflare:
+                    self._cloudflare_solver(page_info.page)
+                    # Make sure the page is fully loaded after the captcha
                     self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                except Exception as e:  # pragma: no cover
-                    log.error(f"Error waiting for selector {params.wait_selector}: {e}")
 
-            page_info.page.wait_for_timeout(params.wait)
+                if params.page_action:
+                    try:
+                        _ = params.page_action(page_info.page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_action: {e}")
+
+                if params.wait_selector:
+                    try:
+                        waiter: Locator = page_info.page.locator(params.wait_selector)
+                        waiter.first.wait_for(state=params.wait_selector_state)
+                        # Wait again after waiting for the selector, helpful with protections like Cloudflare
+                        self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error waiting for selector {params.wait_selector}: {e}")
+
+                page_info.page.wait_for_timeout(params.wait)
+
+                # Create response object
+                response = ResponseFactory.from_playwright_response(
+                    page_info.page, first_response, final_response[0], params.selector_config
+                )
 
-            # Create response object
-            response = ResponseFactory.from_playwright_response(
-                page_info.page, first_response, final_response[0], params.selector_config
-            )
+                # Close the page to free up resources
+                page_info.page.close()
+                self.page_pool.pages.remove(page_info)
 
-            # Close the page to free up resources
-            page_info.page.close()
-            self.page_pool.pages.remove(page_info)
+                return response
 
-            return response
+            except Exception as e:
+                page_info.mark_error()
+                page_info.page.close()
+                self.page_pool.pages.remove(page_info)
 
-        except Exception as e:
-            page_info.mark_error()
-            page_info.page.close()
-            self.page_pool.pages.remove(page_info)
-            raise e
+                if attempt < self._config.retries - 1 and self._is_retriable(e):
+                    log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
+                    time_sleep(self._config.retry_delay)
+                else:
+                    raise
+
+        # For type checking purposes only
+        raise AssertionError("Unreachable: retry loop must return or raise")  # pragma: no cover
 
 
 class AsyncStealthySession(AsyncSession, StealthySessionMixin):
@@ -422,53 +433,62 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
             else None
         )
 
-        page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        final_response = [None]
-        handle_response = self._create_response_handler(page_info, final_response)
-
-        try:
-            # Navigate to URL and wait for a specified state
-            page_info.page.on("response", handle_response)
-            first_response = await page_info.page.goto(url, referer=referer)
-            await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-
-            if not first_response:
-                raise RuntimeError(f"Failed to get response for {url}")
+        for attempt in range(self._config.retries):
+            page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
+            final_response = [None]
+            handle_response = self._create_response_handler(page_info, final_response)
 
-            if params.solve_cloudflare:
-                await self._cloudflare_solver(page_info.page)
-                # Make sure the page is fully loaded after the captcha
+            try:
+                # Navigate to URL and wait for a specified state
+                page_info.page.on("response", handle_response)
+                first_response = await page_info.page.goto(url, referer=referer)
                 await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
 
-            if params.page_action:
-                try:
-                    _ = await params.page_action(page_info.page)
-                except Exception as e:
-                    log.error(f"Error executing page_action: {e}")
-
-            if params.wait_selector:
-                try:
-                    waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
-                    await waiter.first.wait_for(state=params.wait_selector_state)
-                    # Wait again after waiting for the selector, helpful with protections like Cloudflare
+                if not first_response:
+                    raise RuntimeError(f"Failed to get response for {url}")
+
+                if params.solve_cloudflare:
+                    await self._cloudflare_solver(page_info.page)
+                    # Make sure the page is fully loaded after the captcha
                     await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                except Exception as e:
-                    log.error(f"Error waiting for selector {params.wait_selector}: {e}")
-
-            await page_info.page.wait_for_timeout(params.wait)
-
-            # Create response object
-            response = await ResponseFactory.from_async_playwright_response(
-                page_info.page, first_response, final_response[0], params.selector_config
-            )
-
-            # Close the page to free up resources
-            await page_info.page.close()
-            self.page_pool.pages.remove(page_info)
-            return response
-
-        except Exception as e:  # pragma: no cover
-            page_info.mark_error()
-            await page_info.page.close()
-            self.page_pool.pages.remove(page_info)
-            raise e
+
+                if params.page_action:
+                    try:
+                        _ = await params.page_action(page_info.page)
+                    except Exception as e:
+                        log.error(f"Error executing page_action: {e}")
+
+                if params.wait_selector:
+                    try:
+                        waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
+                        await waiter.first.wait_for(state=params.wait_selector_state)
+                        # Wait again after waiting for the selector, helpful with protections like Cloudflare
+                        await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+                    except Exception as e:
+                        log.error(f"Error waiting for selector {params.wait_selector}: {e}")
+
+                await page_info.page.wait_for_timeout(params.wait)
+
+                # Create response object
+                response = await ResponseFactory.from_async_playwright_response(
+                    page_info.page, first_response, final_response[0], params.selector_config
+                )
+
+                # Close the page to free up resources
+                await page_info.page.close()
+                self.page_pool.pages.remove(page_info)
+                return response
+
+            except Exception as e:  # pragma: no cover
+                page_info.mark_error()
+                await page_info.page.close()
+                self.page_pool.pages.remove(page_info)
+
+                if attempt < self._config.retries - 1 and self._is_retriable(e):
+                    log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
+                    await asyncio_sleep(self._config.retry_delay)
+                else:
+                    raise
+
+        # For type checking purposes only
+        raise AssertionError("Unreachable: retry loop must return or raise")  # pragma: no cover

scrapling/engines/_browsers/_types.py

@@ -79,6 +79,8 @@ if TYPE_CHECKING:  # pragma: no cover
         cdp_url: Optional[str]
         useragent: Optional[str]
         extra_flags: Optional[List[str]]
+        retries: int
+        retry_delay: int | float
 
     class PlaywrightFetchParams(TypedDict, total=False):
         load_dom: bool

scrapling/engines/_browsers/_validators.py

@@ -50,6 +50,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
 
 # Type aliases for cleaner annotations
 PagesCount = Annotated[int, Meta(ge=1, le=50)]
+RetriesCount = Annotated[int, Meta(ge=1, le=10)]
 Seconds = Annotated[int, float, Meta(ge=0)]
 
 
@@ -80,6 +81,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     cdp_url: Optional[str] = None
     useragent: Optional[str] = None
     extra_flags: Optional[List[str]] = None
+    retries: RetriesCount = 3
+    retry_delay: Seconds = 1
 
     def __post_init__(self):  # pragma: no cover
         """Custom validation after msgspec validation"""