from io import BytesIO from time import sleep from smolagents.utils import encode_image_base64, make_image_url from smolagents import OpenAIServerModel, CodeAgent from smolagents.agents import ActionStep import helium from PIL import Image def check_reasoning_and_plot( final_answer, agent_memory, map_image_path: str = "src/party_planner/saved_map.png" ) -> bool: final_answer multimodal_model = OpenAIServerModel("gpt-4o", max_tokens=2048) image = Image.open(map_image_path) prompt = ( f"Here is a user-given task and the agent steps: {agent_memory.get_succinct_steps()}. Now here is the plot that was made." "Please check that the reasoning process and plot are correct: do they correctly answer the given task?" "First list reasons why yes/no, then write your final decision: PASS in caps lock if it is satisfactory, FAIL if it is not." "Don't be harsh: if the plot mostly solves the task, it should pass." "To pass, a plot should be made using px.scatter_map and not any other method (scatter_map looks nicer)." ) messages = [ { "role": "user", "content": [ { "type": "text", "text": prompt, }, { "type": "image_url", "image_url": {"url": make_image_url(encode_image_base64(image))}, }, ], } ] output = multimodal_model(messages).content print("Feedback: ", output) if "FAIL" in output: raise Exception(output) return True def save_screenshot(memory_step: ActionStep, agent: CodeAgent) -> None: sleep(1.0) # Let JavaScript animations happen before taking the screenshot driver = helium.get_driver() current_step = memory_step.step_number if driver is not None: for previous_memory_step in agent.memory.steps: # Remove previous screenshots from logs for lean processing if isinstance(previous_memory_step, ActionStep) and previous_memory_step.step_number <= current_step - 2: previous_memory_step.observations_images = None png_bytes = driver.get_screenshot_as_png() image = Image.open(BytesIO(png_bytes)) print(f"Captured a browser screenshot: {image.size} pixels") memory_step.observations_images = [image.copy()] # Create a copy to ensure it persists, important! # Update observations with current URL url_info = f"Current url: {driver.current_url}" memory_step.observations = ( url_info if memory_step.observations is None else memory_step.observations + "\n" + url_info ) return