import pdb from typing import List, Optional from browser_use.agent.prompts import SystemPrompt, AgentMessagePrompt from browser_use.agent.views import ActionResult, ActionModel from browser_use.browser.views import BrowserState from langchain_core.messages import HumanMessage, SystemMessage from datetime import datetime from .custom_views import CustomAgentStepInfo class CustomSystemPrompt(SystemPrompt): def important_rules(self) -> str: """ Returns the important rules for the agent. """ text = r""" 1. RESPONSE FORMAT: You must ALWAYS respond with valid JSON in this exact format: { "current_state": { "prev_action_evaluation": "Success|Failed|Unknown - Analyze the current elements and the image to check if the previous goals/actions are successful like intended by the task. Ignore the action result. The website is the ground truth. Also mention if something unexpected happened like new suggestions in an input field. Shortly state why/why not. Note that the result you output must be consistent with the reasoning you output afterwards. If you consider it to be 'Failed,' you should reflect on this during your thought.", "important_contents": "Output important contents closely related to user\'s instruction on the current page. If there is, please output the contents. If not, please output empty string ''.", "task_progress": "Task Progress is a general summary of the current contents that have been completed. Just summarize the contents that have been actually completed based on the content at current step and the history operations. Please list each completed item individually, such as: 1. Input username. 2. Input Password. 3. Click confirm button. Please return string type not a list.", "future_plans": "Based on the user's request and the current state, outline the remaining steps needed to complete the task. This should be a concise list of actions yet to be performed, such as: 1. Select a date. 2. Choose a specific time slot. 3. Confirm booking. Please return string type not a list.", "thought": "Think about the requirements that have been completed in previous operations and the requirements that need to be completed in the next one operation. If your output of prev_action_evaluation is 'Failed', please reflect and output your reflection here.", "summary": "Please generate a brief natural language description for the operation in next actions based on your Thought." }, "action": [ * actions in sequences, please refer to **Common action sequences**. Each output action MUST be formated as: \{action_name\: action_params\}* ] } 2. ACTIONS: You can specify multiple actions to be executed in sequence. Common action sequences: - Form filling: [ {"input_text": {"index": 1, "text": "username"}}, {"input_text": {"index": 2, "text": "password"}}, {"click_element": {"index": 3}} ] - Navigation and extraction: [ {"go_to_url": {"url": "https://example.com"}}, {"extract_page_content": {}} ] 3. ELEMENT INTERACTION: - Only use indexes that exist in the provided element list - Each element has a unique index number (e.g., "33[:] _[:] Non-interactive text Notes: - Only elements with numeric indexes are interactive - _[:] elements provide context but cannot be interacted with """ def get_system_message(self) -> SystemMessage: """ Get the system prompt for the agent. Returns: str: Formatted system prompt """ AGENT_PROMPT = f"""You are a precise browser automation agent that interacts with websites through structured commands. Your role is to: 1. Analyze the provided webpage elements and structure 2. Plan a sequence of actions to accomplish the given task 3. Your final result MUST be a valid JSON as the **RESPONSE FORMAT** described, containing your action sequence and state assessment, No need extra content to expalin. {self.input_format()} {self.important_rules()} Functions: {self.default_action_description} Remember: Your responses must be valid JSON matching the specified format. Each action in the sequence must be valid.""" return SystemMessage(content=AGENT_PROMPT) class CustomAgentMessagePrompt(AgentMessagePrompt): def __init__( self, state: BrowserState, actions: Optional[List[ActionModel]] = None, result: Optional[List[ActionResult]] = None, include_attributes: list[str] = [], max_error_length: int = 400, step_info: Optional[CustomAgentStepInfo] = None, ): super(CustomAgentMessagePrompt, self).__init__(state=state, result=result, include_attributes=include_attributes, max_error_length=max_error_length, step_info=step_info ) self.actions = actions def get_user_message(self) -> HumanMessage: if self.step_info: step_info_description = f'Current step: {self.step_info.step_number}/{self.step_info.max_steps}\n' else: step_info_description = '' time_str = datetime.now().strftime("%Y-%m-%d %H:%M") step_info_description += f"Current date and time: {time_str}" elements_text = self.state.element_tree.clickable_elements_to_string(include_attributes=self.include_attributes) has_content_above = (self.state.pixels_above or 0) > 0 has_content_below = (self.state.pixels_below or 0) > 0 if elements_text != '': if has_content_above: elements_text = ( f'... {self.state.pixels_above} pixels above - scroll or extract content to see more ...\n{elements_text}' ) else: elements_text = f'[Start of page]\n{elements_text}' if has_content_below: elements_text = ( f'{elements_text}\n... {self.state.pixels_below} pixels below - scroll or extract content to see more ...' ) else: elements_text = f'{elements_text}\n[End of page]' else: elements_text = 'empty page' state_description = f""" {step_info_description} 1. Task: {self.step_info.task}. 2. Hints(Optional): {self.step_info.add_infos} 3. Memory: {self.step_info.memory} 4. Current url: {self.state.url} 5. Available tabs: {self.state.tabs} 6. Interactive elements: {elements_text} """ if self.actions and self.result: state_description += "\n **Previous Actions** \n" state_description += f'Previous step: {self.step_info.step_number-1}/{self.step_info.max_steps} \n' for i, result in enumerate(self.result): action = self.actions[i] state_description += f"Previous action {i + 1}/{len(self.result)}: {action.model_dump_json(exclude_unset=True)}\n" if result.include_in_memory: if result.extracted_content: state_description += f"Result of previous action {i + 1}/{len(self.result)}: {result.extracted_content}\n" if result.error: # only use last 300 characters of error error = result.error[-self.max_error_length:] state_description += ( f"Error of previous action {i + 1}/{len(self.result)}: ...{error}\n" ) if self.state.screenshot: # Format message for vision model return HumanMessage( content=[ {"type": "text", "text": state_description}, { "type": "image_url", "image_url": { "url": f"data:image/png;base64,{self.state.screenshot}" }, }, ] ) return HumanMessage(content=state_description)