Upload 5 files

src/utils/agent_state.py

@@ -0,0 +1,30 @@
+import asyncio
+
+class AgentState:
+    _instance = None
+
+    def __init__(self):
+        if not hasattr(self, '_stop_requested'):
+            self._stop_requested = asyncio.Event()
+            self.last_valid_state = None  # store the last valid browser state
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(AgentState, cls).__new__(cls)
+        return cls._instance
+
+    def request_stop(self):
+        self._stop_requested.set()
+
+    def clear_stop(self):
+        self._stop_requested.clear()
+        self.last_valid_state = None
+
+    def is_stop_requested(self):
+        return self._stop_requested.is_set()
+
+    def set_last_valid_state(self, state):
+        self.last_valid_state = state
+
+    def get_last_valid_state(self):
+        return self.last_valid_state

src/utils/default_config_settings.py

@@ -0,0 +1,122 @@
+import os
+import pickle
+import uuid
+import gradio as gr
+
+
+def default_config():
+    """Prepare the default configuration"""
+    return {
+        "agent_type": "custom",
+        "max_steps": 100,
+        "max_actions_per_step": 10,
+        "use_vision": True,
+        "tool_call_in_content": True,
+        "llm_provider": "openai",
+        "llm_model_name": "gpt-4o",
+        "llm_temperature": 1.0,
+        "llm_base_url": "",
+        "llm_api_key": "",
+        "use_own_browser": os.getenv("CHROME_PERSISTENT_SESSION", "false").lower() == "true",
+        "keep_browser_open": False,
+        "headless": False,
+        "disable_security": True,
+        "enable_recording": True,
+        "window_w": 1280,
+        "window_h": 1100,
+        "save_recording_path": "./tmp/record_videos",
+        "save_trace_path": "./tmp/traces",
+        "save_agent_history_path": "./tmp/agent_history",
+        "task": "go to google.com and type 'OpenAI' click search and give me the first url",
+    }
+
+
+def load_config_from_file(config_file):
+    """Load settings from a UUID.pkl file."""
+    try:
+        with open(config_file, 'rb') as f:
+            settings = pickle.load(f)
+        return settings
+    except Exception as e:
+        return f"Error loading configuration: {str(e)}"
+
+
+def save_config_to_file(settings, save_dir="./tmp/webui_settings"):
+    """Save the current settings to a UUID.pkl file with a UUID name."""
+    os.makedirs(save_dir, exist_ok=True)
+    config_file = os.path.join(save_dir, f"{uuid.uuid4()}.pkl")
+    with open(config_file, 'wb') as f:
+        pickle.dump(settings, f)
+    return f"Configuration saved to {config_file}"
+
+
+def save_current_config(*args):
+    current_config = {
+        "agent_type": args[0],
+        "max_steps": args[1],
+        "max_actions_per_step": args[2],
+        "use_vision": args[3],
+        "tool_call_in_content": args[4],
+        "llm_provider": args[5],
+        "llm_model_name": args[6],
+        "llm_temperature": args[7],
+        "llm_base_url": args[8],
+        "llm_api_key": args[9],
+        "use_own_browser": args[10],
+        "keep_browser_open": args[11],
+        "headless": args[12],
+        "disable_security": args[13],
+        "enable_recording": args[14],
+        "window_w": args[15],
+        "window_h": args[16],
+        "save_recording_path": args[17],
+        "save_trace_path": args[18],
+        "save_agent_history_path": args[19],
+        "task": args[20],
+    }
+    return save_config_to_file(current_config)
+
+
+def update_ui_from_config(config_file):
+    if config_file is not None:
+        loaded_config = load_config_from_file(config_file.name)
+        if isinstance(loaded_config, dict):
+            return (
+                gr.update(value=loaded_config.get("agent_type", "custom")),
+                gr.update(value=loaded_config.get("max_steps", 100)),
+                gr.update(value=loaded_config.get("max_actions_per_step", 10)),
+                gr.update(value=loaded_config.get("use_vision", True)),
+                gr.update(value=loaded_config.get("tool_call_in_content", True)),
+                gr.update(value=loaded_config.get("llm_provider", "openai")),
+                gr.update(value=loaded_config.get("llm_model_name", "gpt-4o")),
+                gr.update(value=loaded_config.get("llm_temperature", 1.0)),
+                gr.update(value=loaded_config.get("llm_base_url", "")),
+                gr.update(value=loaded_config.get("llm_api_key", "")),
+                gr.update(value=loaded_config.get("use_own_browser", False)),
+                gr.update(value=loaded_config.get("keep_browser_open", False)),
+                gr.update(value=loaded_config.get("headless", False)),
+                gr.update(value=loaded_config.get("disable_security", True)),
+                gr.update(value=loaded_config.get("enable_recording", True)),
+                gr.update(value=loaded_config.get("window_w", 1280)),
+                gr.update(value=loaded_config.get("window_h", 1100)),
+                gr.update(value=loaded_config.get("save_recording_path", "./tmp/record_videos")),
+                gr.update(value=loaded_config.get("save_trace_path", "./tmp/traces")),
+                gr.update(value=loaded_config.get("save_agent_history_path", "./tmp/agent_history")),
+                gr.update(value=loaded_config.get("task", "")),
+                "Configuration loaded successfully."
+            )
+        else:
+            return (
+                gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
+                gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
+                gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
+                gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
+                gr.update(), "Error: Invalid configuration file."
+            )
+    return (
+        gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
+        gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
+        gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
+        gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
+        gr.update(), "No file selected."
+    )

src/utils/llm.py

@@ -0,0 +1,101 @@
+from openai import OpenAI
+import pdb
+from langchain_openai import ChatOpenAI
+from langchain_core.globals import get_llm_cache
+from langchain_core.language_models.base import (
+    BaseLanguageModel,
+    LangSmithParams,
+    LanguageModelInput,
+)
+from langchain_core.load import dumpd, dumps
+from langchain_core.messages import (
+    AIMessage,
+    SystemMessage,
+    AnyMessage,
+    BaseMessage,
+    BaseMessageChunk,
+    HumanMessage,
+    convert_to_messages,
+    message_chunk_to_message,
+)
+from langchain_core.outputs import (
+    ChatGeneration,
+    ChatGenerationChunk,
+    ChatResult,
+    LLMResult,
+    RunInfo,
+)
+from langchain_core.output_parsers.base import OutputParserLike
+from langchain_core.runnables import Runnable, RunnableConfig
+from langchain_core.tools import BaseTool
+
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Literal,
+    Optional,
+    Union,
+    cast,
+)
+
+class DeepSeekR1ChatOpenAI(ChatOpenAI):
+    
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(
+            base_url=kwargs.get("base_url"),
+            api_key=kwargs.get("api_key")
+        ) 
+        
+    async def ainvoke(
+        self,
+        input: LanguageModelInput,
+        config: Optional[RunnableConfig] = None,
+        *,
+        stop: Optional[list[str]] = None,
+        **kwargs: Any,
+    ) -> AIMessage:
+        message_history = []
+        for input_ in input:
+            if isinstance(input_, SystemMessage):
+                message_history.append({"role": "system", "content": input_.content})
+            elif isinstance(input_, AIMessage):
+                message_history.append({"role": "assistant", "content": input_.content})
+            else:
+                message_history.append({"role": "user", "content": input_.content})
+        
+        response = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=messages
+        )
+
+        reasoning_content = response.choices[0].message.reasoning_content
+        content = response.choices[0].message.content
+        return AIMessage(content=content, reasoning_content=reasoning_content)
+    
+    def invoke(
+        self,
+        input: LanguageModelInput,
+        config: Optional[RunnableConfig] = None,
+        *,
+        stop: Optional[list[str]] = None,
+        **kwargs: Any,
+    ) -> AIMessage:
+        message_history = []
+        for input_ in input:
+            if isinstance(input_, SystemMessage):
+                message_history.append({"role": "system", "content": input_.content})
+            elif isinstance(input_, AIMessage):
+                message_history.append({"role": "assistant", "content": input_.content})
+            else:
+                message_history.append({"role": "user", "content": input_.content})
+        
+        response = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=message_history
+        )
+
+        reasoning_content = response.choices[0].message.reasoning_content
+        content = response.choices[0].message.content
+        return AIMessage(content=content, reasoning_content=reasoning_content)

src/utils/utils.py

@@ -0,0 +1,207 @@
+import base64
+import os
+import time
+from pathlib import Path
+from typing import Dict, Optional
+
+from langchain_anthropic import ChatAnthropic
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_ollama import ChatOllama
+from langchain_openai import AzureChatOpenAI, ChatOpenAI
+import gradio as gr
+
+from .llm import DeepSeekR1ChatOpenAI
+
+def get_llm_model(provider: str, **kwargs):
+    """
+    获取LLM 模型
+    :param provider: 模型类型
+    :param kwargs:
+    :return:
+    """
+    if provider == "anthropic":
+        if not kwargs.get("base_url", ""):
+            base_url = "https://api.anthropic.com"
+        else:
+            base_url = kwargs.get("base_url")
+
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("ANTHROPIC_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+
+        return ChatAnthropic(
+            model_name=kwargs.get("model_name", "claude-3-5-sonnet-20240620"),
+            temperature=kwargs.get("temperature", 0.0),
+            base_url=base_url,
+            api_key=api_key,
+        )
+    elif provider == "openai":
+        if not kwargs.get("base_url", ""):
+            base_url = os.getenv("OPENAI_ENDPOINT", "https://api.openai.com/v1")
+        else:
+            base_url = kwargs.get("base_url")
+
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("OPENAI_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+
+        return ChatOpenAI(
+            model=kwargs.get("model_name", "gpt-4o"),
+            temperature=kwargs.get("temperature", 0.0),
+            base_url=base_url,
+            api_key=api_key,
+        )
+    elif provider == "deepseek":
+        if not kwargs.get("base_url", ""):
+            base_url = os.getenv("DEEPSEEK_ENDPOINT", "")
+        else:
+            base_url = kwargs.get("base_url")
+
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("DEEPSEEK_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+
+        if kwargs.get("model_name", "deepseek-chat") == "deepseek-reasoner":
+            return DeepSeekR1ChatOpenAI(
+                model=kwargs.get("model_name", "deepseek-reasoner"),
+                temperature=kwargs.get("temperature", 0.0),
+                base_url=base_url,
+                api_key=api_key,
+            )
+        else:
+            return ChatOpenAI(
+                model=kwargs.get("model_name", "deepseek-chat"),
+                temperature=kwargs.get("temperature", 0.0),
+                base_url=base_url,
+                api_key=api_key,
+            )
+    elif provider == "gemini":
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("GOOGLE_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+        return ChatGoogleGenerativeAI(
+            model=kwargs.get("model_name", "gemini-2.0-flash-exp"),
+            temperature=kwargs.get("temperature", 0.0),
+            google_api_key=api_key,
+        )
+    elif provider == "ollama":
+        return ChatOllama(
+            model=kwargs.get("model_name", "qwen2.5:7b"),
+            temperature=kwargs.get("temperature", 0.0),
+            num_ctx=kwargs.get("num_ctx", 32000),
+            base_url=kwargs.get("base_url", "http://localhost:11434"),
+        )
+    elif provider == "azure_openai":
+        if not kwargs.get("base_url", ""):
+            base_url = os.getenv("AZURE_OPENAI_ENDPOINT", "")
+        else:
+            base_url = kwargs.get("base_url")
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("AZURE_OPENAI_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+        return AzureChatOpenAI(
+            model=kwargs.get("model_name", "gpt-4o"),
+            temperature=kwargs.get("temperature", 0.0),
+            api_version="2024-05-01-preview",
+            azure_endpoint=base_url,
+            api_key=api_key,
+        )
+    else:
+        raise ValueError(f"Unsupported provider: {provider}")
+    
+# Predefined model names for common providers
+model_names = {
+    "anthropic": ["claude-3-5-sonnet-20240620", "claude-3-opus-20240229"],
+    "openai": ["gpt-4o", "gpt-4", "gpt-3.5-turbo"],
+    "deepseek": ["deepseek-chat", "deepseek-reasoner"],
+    "gemini": ["gemini-2.0-flash-exp", "gemini-2.0-flash-thinking-exp", "gemini-1.5-flash-latest", "gemini-1.5-flash-8b-latest", "gemini-2.0-flash-thinking-exp-1219" ],
+    "ollama": ["qwen2.5:7b", "llama2:7b"],
+    "azure_openai": ["gpt-4o", "gpt-4", "gpt-3.5-turbo"]
+}
+
+# Callback to update the model name dropdown based on the selected provider
+def update_model_dropdown(llm_provider, api_key=None, base_url=None):
+    """
+    Update the model name dropdown with predefined models for the selected provider.
+    """
+    # Use API keys from .env if not provided
+    if not api_key:
+        api_key = os.getenv(f"{llm_provider.upper()}_API_KEY", "")
+    if not base_url:
+        base_url = os.getenv(f"{llm_provider.upper()}_BASE_URL", "")
+
+    # Use predefined models for the selected provider
+    if llm_provider in model_names:
+        return gr.Dropdown(choices=model_names[llm_provider], value=model_names[llm_provider][0], interactive=True)
+    else:
+        return gr.Dropdown(choices=[], value="", interactive=True, allow_custom_value=True)
+        
+def encode_image(img_path):
+    if not img_path:
+        return None
+    with open(img_path, "rb") as fin:
+        image_data = base64.b64encode(fin.read()).decode("utf-8")
+    return image_data
+
+
+def get_latest_files(directory: str, file_types: list = ['.webm', '.zip']) -> Dict[str, Optional[str]]:
+    """Get the latest recording and trace files"""
+    latest_files: Dict[str, Optional[str]] = {ext: None for ext in file_types}
+    
+    if not os.path.exists(directory):
+        os.makedirs(directory, exist_ok=True)
+        return latest_files
+
+    for file_type in file_types:
+        try:
+            matches = list(Path(directory).rglob(f"*{file_type}"))
+            if matches:
+                latest = max(matches, key=lambda p: p.stat().st_mtime)
+                # Only return files that are complete (not being written)
+                if time.time() - latest.stat().st_mtime > 1.0:
+                    latest_files[file_type] = str(latest)
+        except Exception as e:
+            print(f"Error getting latest {file_type} file: {e}")
+            
+    return latest_files
+async def capture_screenshot(browser_context):
+    """Capture and encode a screenshot"""
+    # Extract the Playwright browser instance
+    playwright_browser = browser_context.browser.playwright_browser  # Ensure this is correct.
+
+    # Check if the browser instance is valid and if an existing context can be reused
+    if playwright_browser and playwright_browser.contexts:
+        playwright_context = playwright_browser.contexts[0]
+    else:
+        return None
+
+    # Access pages in the context
+    pages = None
+    if playwright_context:
+        pages = playwright_context.pages
+
+    # Use an existing page or create a new one if none exist
+    if pages:
+        active_page = pages[0]
+        for page in pages:
+            if page.url != "about:blank":
+                active_page = page
+    else:
+        return None
+
+    # Take screenshot
+    try:
+        screenshot = await active_page.screenshot(
+            type='jpeg',
+            quality=75,
+            scale="css"
+        )
+        encoded = base64.b64encode(screenshot).decode('utf-8')
+        return encoded
+    except Exception as e:
+        return None