feat: Add an mcp server

scrapling/cli.py

@@ -133,6 +133,13 @@ def install(force):
         print("The dependencies are already installed")
 
 
+@command(help="Run Scrapling's MCP server (Check the docs for more info).")
+def mcp():
+    from scrapling.core.ai import ScraplingMCPServer
+
+    ScraplingMCPServer().serve()
+
+
 @command(help="Interactive scraping console")
 @option(
     "-c",
@@ -824,3 +831,4 @@ def main():
 main.add_command(install)
 main.add_command(shell)
 main.add_command(extract)
+main.add_command(mcp)

scrapling/core/ai.py

@@ -0,0 +1,613 @@
+from asyncio import gather
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import BaseModel, Field
+
+from scrapling.core.shell import Convertor
+from scrapling.engines.toolbelt import Response as _ScraplingResponse
+from scrapling.fetchers import (
+    Fetcher,
+    FetcherSession,
+    DynamicFetcher,
+    AsyncDynamicSession,
+    StealthyFetcher,
+    AsyncStealthySession,
+)
+from scrapling.core._types import (
+    Optional,
+    Literal,
+    Tuple,
+    extraction_types,
+    Union,
+    Mapping,
+    Dict,
+    List,
+    SelectorWaitStates,
+    Generator,
+)
+from curl_cffi.requests import (
+    BrowserTypeLiteral,
+)
+
+
+class ResponseModel(BaseModel):
+    """Request's response information structure."""
+
+    status: int = Field(description="The status code returned by the website.")
+    content: list[str] = Field(
+        description="The content as Markdown/HTML or the text content of the page."
+    )
+    url: str = Field(
+        description="The URL given by the user that resulted in this response."
+    )
+
+
+def _ContentTranslator(
+    content: Generator[str, None, None], page: _ScraplingResponse
+) -> ResponseModel:
+    """Convert a content generator to a list of ResponseModel objects."""
+    return ResponseModel(
+        status=page.status, content=[result for result in content], url=page.url
+    )
+
+
+class ScraplingMCPServer:
+    _server = FastMCP(name="Scrapling")
+
+    @staticmethod
+    @_server.tool()
+    def get(
+        url: str,
+        impersonate: Optional[BrowserTypeLiteral] = "chrome",
+        extraction_type: extraction_types = "markdown",
+        css_selector: Optional[str] = None,
+        main_content_only: bool = True,
+        params: Optional[Union[Dict, List, Tuple]] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = None,
+        cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None,
+        timeout: Optional[Union[int, float]] = 30,
+        follow_redirects: bool = True,
+        max_redirects: int = 30,
+        retries: Optional[int] = 3,
+        retry_delay: Optional[int] = 1,
+        proxy: Optional[str] = None,
+        proxy_auth: Optional[Tuple[str, str]] = None,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = True,
+        http3: Optional[bool] = False,
+        stealthy_headers: Optional[bool] = True,
+    ) -> ResponseModel:
+        """Make GET HTTP request to a URL and return a structured output of the result.
+        Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly.
+        Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
+
+        :param url: The URL to request.
+        :param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default.
+        :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
+            - Markdown will convert the page content to Markdown format.
+            - HTML will return the raw HTML content of the page.
+            - Text will return the text content of the page.
+        :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
+        :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
+        :param params: Query string parameters for the request.
+        :param headers: Headers to include in the request.
+        :param cookies: Cookies to use in the request.
+        :param timeout: Number of seconds to wait before timing out.
+        :param follow_redirects: Whether to follow redirects. Defaults to True.
+        :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+        :param retries: Number of retry attempts. Defaults to 3.
+        :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+        :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+                     Cannot be used together with the `proxies` parameter.
+        :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+        :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+        :param verify: Whether to verify HTTPS certificates.
+        :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
+        :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
+        """
+        page = Fetcher.get(
+            url,
+            auth=auth,
+            proxy=proxy,
+            http3=http3,
+            verify=verify,
+            params=params,
+            proxy_auth=proxy_auth,
+            retry_delay=retry_delay,
+            stealthy_headers=stealthy_headers,
+            impersonate=impersonate,
+            headers=headers,
+            cookies=cookies,
+            timeout=timeout,
+            retries=retries,
+            max_redirects=max_redirects,
+            follow_redirects=follow_redirects,
+        )
+        return _ContentTranslator(
+            Convertor._extract_content(
+                page,
+                css_selector=css_selector,
+                extraction_type=extraction_type,
+                main_content_only=main_content_only,
+            ),
+            page,
+        )
+
+    @staticmethod
+    @_server.tool()
+    async def bulk_get(
+        urls: Tuple[str, ...],
+        impersonate: Optional[BrowserTypeLiteral] = "chrome",
+        extraction_type: extraction_types = "markdown",
+        css_selector: Optional[str] = None,
+        main_content_only: bool = True,
+        params: Optional[Union[Dict, List, Tuple]] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = None,
+        cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None,
+        timeout: Optional[Union[int, float]] = 30,
+        follow_redirects: bool = True,
+        max_redirects: int = 30,
+        retries: Optional[int] = 3,
+        retry_delay: Optional[int] = 1,
+        proxy: Optional[str] = None,
+        proxy_auth: Optional[Tuple[str, str]] = None,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = True,
+        http3: Optional[bool] = False,
+        stealthy_headers: Optional[bool] = True,
+    ) -> List[ResponseModel]:
+        """Make GET HTTP request to a group of URLs and for each URL, return a structured output of the result.
+        Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly.
+        Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
+
+        :param urls: A tuple of the URLs to request.
+        :param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default.
+        :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
+            - Markdown will convert the page content to Markdown format.
+            - HTML will return the raw HTML content of the page.
+            - Text will return the text content of the page.
+        :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
+        :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
+        :param params: Query string parameters for the request.
+        :param headers: Headers to include in the request.
+        :param cookies: Cookies to use in the request.
+        :param timeout: Number of seconds to wait before timing out.
+        :param follow_redirects: Whether to follow redirects. Defaults to True.
+        :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+        :param retries: Number of retry attempts. Defaults to 3.
+        :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+        :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+                     Cannot be used together with the `proxies` parameter.
+        :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+        :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+        :param verify: Whether to verify HTTPS certificates.
+        :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
+        :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
+        """
+        async with FetcherSession() as session:
+            tasks = [
+                session.get(
+                    url,
+                    auth=auth,
+                    proxy=proxy,
+                    http3=http3,
+                    verify=verify,
+                    params=params,
+                    headers=headers,
+                    cookies=cookies,
+                    timeout=timeout,
+                    retries=retries,
+                    proxy_auth=proxy_auth,
+                    retry_delay=retry_delay,
+                    impersonate=impersonate,
+                    max_redirects=max_redirects,
+                    follow_redirects=follow_redirects,
+                    stealthy_headers=stealthy_headers,
+                )
+                for url in urls
+            ]
+            responses = await gather(*tasks)
+            return [
+                _ContentTranslator(
+                    Convertor._extract_content(
+                        page,
+                        css_selector=css_selector,
+                        extraction_type=extraction_type,
+                        main_content_only=main_content_only,
+                    ),
+                    page,
+                )
+                for page in responses
+            ]
+
+    @staticmethod
+    @_server.tool()
+    async def fetch(
+        url: str,
+        extraction_type: extraction_types = "markdown",
+        css_selector: Optional[str] = None,
+        main_content_only: bool = True,
+        headless: bool = False,
+        google_search: bool = True,
+        hide_canvas: bool = False,
+        disable_webgl: bool = False,
+        real_chrome: bool = False,
+        stealth: bool = False,
+        wait: Union[int, float] = 0,
+        proxy: Optional[Union[str, Dict[str, str]]] = None,
+        locale: str = "en-US",
+        extra_headers: Optional[Dict[str, str]] = None,
+        useragent: Optional[str] = None,
+        cdp_url: Optional[str] = None,
+        timeout: Union[int, float] = 30000,
+        disable_resources: bool = False,
+        wait_selector: Optional[str] = None,
+        cookies: Optional[List[Dict]] = None,
+        network_idle: bool = False,
+        wait_selector_state: SelectorWaitStates = "attached",
+    ) -> ResponseModel:
+        """Use playwright to open a browser to fetch a URL and return a structured output of the result.
+        Note: This is only suitable for low-mid protection levels.
+        Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
+
+        :param url: The URL to request.
+        :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
+            - Markdown will convert the page content to Markdown format.
+            - HTML will return the raw HTML content of the page.
+            - Text will return the text content of the page.
+        :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
+        :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
+        :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
+        :param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+        :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+        :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+        :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
+        :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
+        :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        """
+        page = await DynamicFetcher.async_fetch(
+            url,
+            wait=wait,
+            proxy=proxy,
+            locale=locale,
+            timeout=timeout,
+            cookies=cookies,
+            stealth=stealth,
+            cdp_url=cdp_url,
+            headless=headless,
+            useragent=useragent,
+            hide_canvas=hide_canvas,
+            real_chrome=real_chrome,
+            network_idle=network_idle,
+            wait_selector=wait_selector,
+            disable_webgl=disable_webgl,
+            extra_headers=extra_headers,
+            google_search=google_search,
+            disable_resources=disable_resources,
+            wait_selector_state=wait_selector_state,
+        )
+        return _ContentTranslator(
+            Convertor._extract_content(
+                page,
+                css_selector=css_selector,
+                extraction_type=extraction_type,
+                main_content_only=main_content_only,
+            ),
+            page,
+        )
+
+    @staticmethod
+    @_server.tool()
+    async def bulk_fetch(
+        urls: Tuple[str, ...],
+        extraction_type: extraction_types = "markdown",
+        css_selector: Optional[str] = None,
+        main_content_only: bool = True,
+        headless: bool = False,
+        google_search: bool = True,
+        hide_canvas: bool = False,
+        disable_webgl: bool = False,
+        real_chrome: bool = False,
+        stealth: bool = False,
+        wait: Union[int, float] = 0,
+        proxy: Optional[Union[str, Dict[str, str]]] = None,
+        locale: str = "en-US",
+        extra_headers: Optional[Dict[str, str]] = None,
+        useragent: Optional[str] = None,
+        cdp_url: Optional[str] = None,
+        timeout: Union[int, float] = 30000,
+        disable_resources: bool = False,
+        wait_selector: Optional[str] = None,
+        cookies: Optional[List[Dict]] = None,
+        network_idle: bool = False,
+        wait_selector_state: SelectorWaitStates = "attached",
+    ) -> List[ResponseModel]:
+        """Use playwright to open a browser, then fetch a group of URLs at the same time, and for each page return a structured output of the result.
+        Note: This is only suitable for low-mid protection levels.
+        Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
+
+        :param urls: A tuple of the URLs to request.
+        :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
+            - Markdown will convert the page content to Markdown format.
+            - HTML will return the raw HTML content of the page.
+            - Text will return the text content of the page.
+        :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
+        :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
+        :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
+        :param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+        :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+        :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+        :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
+        :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
+        :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        """
+        async with AsyncDynamicSession(
+            wait=wait,
+            proxy=proxy,
+            locale=locale,
+            timeout=timeout,
+            cookies=cookies,
+            stealth=stealth,
+            cdp_url=cdp_url,
+            headless=headless,
+            max_pages=len(urls),
+            useragent=useragent,
+            hide_canvas=hide_canvas,
+            real_chrome=real_chrome,
+            network_idle=network_idle,
+            wait_selector=wait_selector,
+            google_search=google_search,
+            disable_webgl=disable_webgl,
+            extra_headers=extra_headers,
+            disable_resources=disable_resources,
+            wait_selector_state=wait_selector_state,
+        ) as session:
+            tasks = [session.fetch(url) for url in urls]
+            responses = await gather(*tasks)
+            return [
+                _ContentTranslator(
+                    Convertor._extract_content(
+                        page,
+                        css_selector=css_selector,
+                        extraction_type=extraction_type,
+                        main_content_only=main_content_only,
+                    ),
+                    page,
+                )
+                for page in responses
+            ]
+
+    @staticmethod
+    @_server.tool()
+    async def stealthy_fetch(
+        url: str,
+        extraction_type: extraction_types = "markdown",
+        css_selector: Optional[str] = None,
+        main_content_only: bool = True,
+        headless: Union[bool, Literal["virtual"]] = True,  # noqa: F821
+        block_images: bool = False,
+        disable_resources: bool = False,
+        block_webrtc: bool = False,
+        allow_webgl: bool = True,
+        network_idle: bool = False,
+        humanize: Union[bool, float] = True,
+        solve_cloudflare: bool = False,
+        wait: Union[int, float] = 0,
+        timeout: Union[int, float] = 30000,
+        wait_selector: Optional[str] = None,
+        addons: Optional[List[str]] = None,
+        wait_selector_state: SelectorWaitStates = "attached",
+        cookies: Optional[List[Dict]] = None,
+        google_search: bool = True,
+        extra_headers: Optional[Dict[str, str]] = None,
+        proxy: Optional[Union[str, Dict[str, str]]] = None,
+        os_randomize: bool = False,
+        disable_ads: bool = False,
+        geoip: bool = False,
+        additional_arguments: Optional[Dict] = None,
+    ) -> ResponseModel:
+        """Use Scrapling's version of the Camoufox browser to fetch a URL and return a structured output of the result.
+        Note: This is best suitable for high protection levels. It's slower than the other tools.
+        Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
+
+        :param url: The URL to request.
+        :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
+            - Markdown will convert the page content to Markdown format.
+            - HTML will return the raw HTML content of the page.
+            - Text will return the text content of the page.
+        :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
+        :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
+        :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+        :param block_images: Prevent the loading of images through Firefox preferences.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param block_webrtc: Blocks WebRTC entirely.
+        :param cookies: Set cookies for the next request.
+        :param addons: List of Firefox addons to use. Must be paths to extracted addons.
+        :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
+        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+        :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
+        :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
+        :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+        :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
+            It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        :param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
+        """
+        page = await StealthyFetcher.async_fetch(
+            url,
+            wait=wait,
+            proxy=proxy,
+            geoip=geoip,
+            addons=addons,
+            timeout=timeout,
+            cookies=cookies,
+            headless=headless,
+            humanize=humanize,
+            allow_webgl=allow_webgl,
+            disable_ads=disable_ads,
+            network_idle=network_idle,
+            block_images=block_images,
+            block_webrtc=block_webrtc,
+            os_randomize=os_randomize,
+            wait_selector=wait_selector,
+            google_search=google_search,
+            extra_headers=extra_headers,
+            solve_cloudflare=solve_cloudflare,
+            disable_resources=disable_resources,
+            wait_selector_state=wait_selector_state,
+            additional_arguments=additional_arguments,
+        )
+        return _ContentTranslator(
+            Convertor._extract_content(
+                page,
+                css_selector=css_selector,
+                extraction_type=extraction_type,
+                main_content_only=main_content_only,
+            ),
+            page,
+        )
+
+    @staticmethod
+    @_server.tool()
+    async def bulk_stealthy_fetch(
+        urls: Tuple[str, ...],
+        extraction_type: extraction_types = "markdown",
+        css_selector: Optional[str] = None,
+        main_content_only: bool = True,
+        headless: Union[bool, Literal["virtual"]] = True,  # noqa: F821
+        block_images: bool = False,
+        disable_resources: bool = False,
+        block_webrtc: bool = False,
+        allow_webgl: bool = True,
+        network_idle: bool = False,
+        humanize: Union[bool, float] = True,
+        solve_cloudflare: bool = False,
+        wait: Union[int, float] = 0,
+        timeout: Union[int, float] = 30000,
+        wait_selector: Optional[str] = None,
+        addons: Optional[List[str]] = None,
+        wait_selector_state: SelectorWaitStates = "attached",
+        cookies: Optional[List[Dict]] = None,
+        google_search: bool = True,
+        extra_headers: Optional[Dict[str, str]] = None,
+        proxy: Optional[Union[str, Dict[str, str]]] = None,
+        os_randomize: bool = False,
+        disable_ads: bool = False,
+        geoip: bool = False,
+        additional_arguments: Optional[Dict] = None,
+    ) -> List[ResponseModel]:
+        """Use Scrapling's version of the Camoufox browser to fetch a group of URLs at the same time, and for each page return a structured output of the result.
+        Note: This is best suitable for high protection levels. It's slower than the other tools.
+        Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
+
+        :param urls: A tuple of the URLs to request.
+        :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
+            - Markdown will convert the page content to Markdown format.
+            - HTML will return the raw HTML content of the page.
+            - Text will return the text content of the page.
+        :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
+        :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
+        :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+        :param block_images: Prevent the loading of images through Firefox preferences.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param block_webrtc: Blocks WebRTC entirely.
+        :param cookies: Set cookies for the next request.
+        :param addons: List of Firefox addons to use. Must be paths to extracted addons.
+        :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
+        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+        :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
+        :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
+        :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+        :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
+            It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        :param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
+        """
+        async with AsyncStealthySession(
+            wait=wait,
+            proxy=proxy,
+            geoip=geoip,
+            addons=addons,
+            timeout=timeout,
+            cookies=cookies,
+            headless=headless,
+            humanize=humanize,
+            max_pages=len(urls),
+            allow_webgl=allow_webgl,
+            disable_ads=disable_ads,
+            block_images=block_images,
+            block_webrtc=block_webrtc,
+            network_idle=network_idle,
+            os_randomize=os_randomize,
+            wait_selector=wait_selector,
+            google_search=google_search,
+            extra_headers=extra_headers,
+            solve_cloudflare=solve_cloudflare,
+            disable_resources=disable_resources,
+            wait_selector_state=wait_selector_state,
+            additional_arguments=additional_arguments,
+        ) as session:
+            tasks = [session.fetch(url) for url in urls]
+            responses = await gather(*tasks)
+            return [
+                _ContentTranslator(
+                    Convertor._extract_content(
+                        page,
+                        css_selector=css_selector,
+                        extraction_type=extraction_type,
+                        main_content_only=main_content_only,
+                    ),
+                    page,
+                )
+                for page in responses
+            ]
+
+    def serve(self):
+        """Serve the MCP server."""
+        self._server.run(transport="stdio")

scrapling/core/shell.py

@@ -36,6 +36,7 @@ from scrapling.core._types import (
     Any,
     Union,
     extraction_types,
+    Generator,
 )
 from scrapling.fetchers import (
     Fetcher,
@@ -589,7 +590,7 @@ class Convertor:
         extraction_type: extraction_types = "markdown",
         css_selector: Optional[str] = None,
         main_content_only: bool = False,
-    ) -> str:
+    ) -> Generator[str, None, None]:
         """Extract the content of an Adaptor"""
         if not page or not isinstance(page, Adaptor):
             raise TypeError("Input must be of type `Adaptor`")
@@ -599,24 +600,25 @@ class Convertor:
             if main_content_only:
                 page = page.css_first("body") or page
 
-            page = page if not css_selector else page.css_first(css_selector)
-            match extraction_type:
-                case "markdown":
-                    return cls._convert_to_markdown(page.body)
-                case "html":
-                    return page.body
-                case "text":
-                    txt_content = page.get_all_text(strip=True)
-                    for s in (
-                        "\n",
-                        "\r",
-                        "\t",
-                        " ",
-                    ):
-                        # Remove consecutive white-spaces
-                        txt_content = re_sub(f"[{s}]+", s, txt_content)
-                    return txt_content
-            return ""
+            pages = [page] if not css_selector else page.css(css_selector)
+            for page in pages:
+                match extraction_type:
+                    case "markdown":
+                        yield cls._convert_to_markdown(page.body)
+                    case "html":
+                        yield page.body
+                    case "text":
+                        txt_content = page.get_all_text(strip=True)
+                        for s in (
+                            "\n",
+                            "\r",
+                            "\t",
+                            " ",
+                        ):
+                            # Remove consecutive white-spaces
+                            txt_content = re_sub(f"[{s}]+", s, txt_content)
+                        yield txt_content
+            yield ""
 
     @classmethod
     def write_content_to_file(
@@ -635,7 +637,11 @@ class Convertor:
             with open(filename, "w", encoding="utf-8") as f:
                 extension = filename.split(".")[-1]
                 f.write(
-                    cls._extract_content(
-                        page, cls._extension_map[extension], css_selector=css_selector
+                    "".join(
+                        cls._extract_content(
+                            page,
+                            cls._extension_map[extension],
+                            css_selector=css_selector,
+                        )
                     )
                 )