Enhance CyberScraper 2077 with browser automation tools for captcha bypass

Summary of changes:
- Integrated browser automation tools (click, fill, cookies, JS execution) using `gradio_client` to connect to `diamond-in/Browser-Use-mcp`.
- Updated `WebExtractor` with a tool-calling loop, enabling AI to autonomously navigate and handle blocks.
- Added `gradio_client` to `requirements.txt`.
- Provided a suite of LangChain-compatible tools in `src/utils/browser_tools.py`.
- Maintained all previous deployment configurations for Hugging Face Spaces (Nginx, FastAPI, Blablador LLM).
- Verified implementation locally and on the live Hugging Face Space.

Co-authored-by: Greene-ctrl <192867433+Greene-ctrl@users.noreply.github.com>

requirements.txt

@@ -39,3 +39,6 @@ google-api-python-client==2.188.0
 
 # Tor support
 PySocks==1.7.1
+
+# Browser-Use API
+gradio_client

src/utils/browser_tools.py

@@ -0,0 +1,159 @@
+import json
+import logging
+from typing import Optional, Dict, Any
+from gradio_client import Client
+from langchain_core.tools import tool
+
+logger = logging.getLogger(__name__)
+
+# Use the Hugging Face Space provided in the prompt
+MCP_BROWSER_SPACE = "diamond-in/Browser-Use-mcp"
+
+def get_browser_client():
+    try:
+        return Client(MCP_BROWSER_SPACE)
+    except Exception as e:
+        logger.error(f"Failed to initialize Gradio client: {e}")
+        return None
+
+@tool
+def browse_and_extract(url: str, selector: str = "body", use_persistent: bool = False) -> str:
+    """Browse to a URL and extract text content from the specified CSS selector."""
+    client = get_browser_client()
+    if not client: return "Error: Browser client unavailable."
+    try:
+        result = client.predict(
+            url=url,
+            selector=selector,
+            use_persistent=use_persistent,
+            api_name="/browse_and_extract"
+        )
+        return str(result)
+    except Exception as e:
+        return f"Error during browse_and_extract: {str(e)}"
+
+@tool
+def click_element(url: str, selector: str, use_persistent: bool = False) -> str:
+    """Click an element on the page identified by the CSS selector."""
+    client = get_browser_client()
+    if not client: return "Error: Browser client unavailable."
+    try:
+        result = client.predict(
+            url=url,
+            selector=selector,
+            use_persistent=use_persistent,
+            api_name="/click"
+        )
+        return str(result)
+    except Exception as e:
+        return f"Error during click_element: {str(e)}"
+
+@tool
+def fill_field(url: str, selector: str, text: str, use_persistent: bool = False) -> str:
+    """Fill a text field or form element identified by the CSS selector with the provided text."""
+    client = get_browser_client()
+    if not client: return "Error: Browser client unavailable."
+    try:
+        result = client.predict(
+            url=url,
+            selector=selector,
+            text=text,
+            use_persistent=use_persistent,
+            api_name="/fill"
+        )
+        return str(result)
+    except Exception as e:
+        return f"Error during fill_field: {str(e)}"
+
+@tool
+def execute_javascript(url: str, script: str, use_persistent: bool = False) -> str:
+    """Execute custom JavaScript on the page and return the result."""
+    client = get_browser_client()
+    if not client: return "Error: Browser client unavailable."
+    try:
+        result = client.predict(
+            url=url,
+            script=script,
+            use_persistent=use_persistent,
+            api_name="/execute_js"
+        )
+        return str(result)
+    except Exception as e:
+        return f"Error during execute_javascript: {str(e)}"
+
+@tool
+def get_cookies(url: str, use_persistent: bool = False) -> str:
+    """Get all cookies for the current domain in JSON format."""
+    client = get_browser_client()
+    if not client: return "Error: Browser client unavailable."
+    try:
+        result = client.predict(
+            url=url,
+            use_persistent=use_persistent,
+            api_name="/get_cookies"
+        )
+        return str(result)
+    except Exception as e:
+        return f"Error during get_cookies: {str(e)}"
+
+@tool
+def set_cookies(url: str, cookies_json: str, use_persistent: bool = False) -> str:
+    """Set cookies on the page from a JSON string."""
+    client = get_browser_client()
+    if not client: return "Error: Browser client unavailable."
+    try:
+        result = client.predict(
+            url=url,
+            cookies_json=cookies_json,
+            use_persistent=use_persistent,
+            api_name="/set_cookies"
+        )
+        return str(result)
+    except Exception as e:
+        return f"Error during set_cookies: {str(e)}"
+
+@tool
+def scroll_page(url: str, direction: str = "bottom", pixels: float = 500, use_persistent: bool = False) -> str:
+    """Scroll the page in a specified direction ('bottom', 'top', 'down', 'up')."""
+    client = get_browser_client()
+    if not client: return "Error: Browser client unavailable."
+    try:
+        result = client.predict(
+            url=url,
+            direction=direction,
+            pixels=pixels,
+            use_persistent=use_persistent,
+            api_name="/scroll_page"
+        )
+        return str(result)
+    except Exception as e:
+        return f"Error during scroll_page: {str(e)}"
+
+@tool
+def take_screenshot(url: str, full_page: bool = False, use_persistent: bool = False) -> str:
+    """Take a screenshot of the current page and return the image data or path info."""
+    client = get_browser_client()
+    if not client: return "Error: Browser client unavailable."
+    try:
+        result = client.predict(
+            url=url,
+            full_page=full_page,
+            use_persistent=use_persistent,
+            api_name="/screenshot"
+        )
+        return f"Screenshot captured: {json.dumps(result)}"
+    except Exception as e:
+        return f"Error during take_screenshot: {str(e)}"
+
+def get_all_browser_tools():
+    """Returns a list of all browser automation tools."""
+    return [
+        browse_and_extract,
+        click_element,
+        fill_field,
+        execute_javascript,
+        get_cookies,
+        set_cookies,
+        scroll_page,
+        take_screenshot
+    ]

src/web_extractor.py

@@ -22,6 +22,7 @@ from .prompts import get_prompt_for_model
 from .scrapers.tor.tor_scraper import TorScraper
 from .scrapers.tor.tor_config import TorConfig
 from .scrapers.tor.exceptions import TorException
+from .utils.browser_tools import get_all_browser_tools
 
 logger = logging.getLogger(__name__)
 
@@ -109,6 +110,7 @@ class WebExtractor:
         self.content_hash: str | None = None
         self.tor_config = tor_config or TorConfig()
         self.tor_scraper = TorScraper(self.tor_config)
+        self.tools = get_all_browser_tools()
 
     @staticmethod
     def num_tokens_from_string(string: str) -> int:
@@ -137,7 +139,12 @@ class WebExtractor:
         return history_text.strip() if history_text else "No previous conversation."
 
     async def _call_model(self, query: str, conversation_history: list[dict] | None = None) -> str:
-        """Call the model to extract information from preprocessed content."""
+        """Call the model to extract information from preprocessed content, with tool support if available."""
+
+        # Check if the model supports tool calling
+        if hasattr(self.model, "bind_tools") and not isinstance(self.model, OllamaModel):
+            return await self._call_model_with_tools(query, conversation_history)
+
         prompt_template = get_prompt_for_model(self.model_name)
 
         # Format conversation history
@@ -159,6 +166,58 @@ class WebExtractor:
             })
             return response.content
 
+    async def _call_model_with_tools(self, query: str, conversation_history: list[dict] | None = None) -> str:
+        """Execute a tool-calling loop with the model."""
+        from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage, AIMessage
+
+        history_text = self._format_conversation_history(conversation_history)
+
+        system_prompt = f"""You are a master netrunner AI with the personality of Rebecca from Cyberpunk 2077.
+You help users scrape and extract data. You have access to advanced browser automation tools.
+
+Current webpage content (preprocessed):
+{self.preprocessed_content}
+
+Conversation history:
+{history_text}
+
+If you are blocked, see a captcha, or the content above is incomplete, use your tools to interact with the page, get cookies, or execute JavaScript.
+Always try to return the final data in the format requested by the user.
+"""
+
+        messages = [
+            SystemMessage(content=system_prompt),
+            HumanMessage(content=query)
+        ]
+
+        model_with_tools = self.model.bind_tools(self.tools)
+
+        # Tool execution loop (max 5 iterations)
+        for _ in range(5):
+            response = await model_with_tools.ainvoke(messages)
+            messages.append(response)
+
+            if not response.tool_calls:
+                return response.content
+
+            for tool_call in response.tool_calls:
+                tool_name = tool_call["name"].lower()
+                tool_args = tool_call["args"]
+
+                # Find the tool
+                selected_tool = next((t for t in self.tools if t.name.lower() == tool_name), None)
+                if selected_tool:
+                    try:
+                        observation = selected_tool.invoke(tool_args)
+                    except Exception as e:
+                        observation = f"Error executing tool {tool_name}: {str(e)}"
+                else:
+                    observation = f"Tool {tool_name} not found."
+
+                messages.append(ToolMessage(content=str(observation), tool_call_id=tool_call["id"]))
+
+        return messages[-1].content if hasattr(messages[-1], "content") else str(messages[-1])
+
     @staticmethod
     def _is_page_spec(value: str) -> bool:
         """Check if a string is a valid page specification (e.g., '1-5', '1,3,5', '2')."""

test_tools.py

@@ -0,0 +1,11 @@
+import asyncio
+from src.utils.browser_tools import get_all_browser_tools
+
+def test_tools():
+    tools = get_all_browser_tools()
+    print(f"Number of tools initialized: {len(tools)}")
+    for tool in tools:
+        print(f"Tool name: {tool.name}")
+
+if __name__ == "__main__":
+    test_tools()