Add files using upload-large-folder tool

.gitattributes

@@ -1 +1,41 @@
 approach/ovod/APE/ape_d_model_final.pth filter=lfs diff=lfs merge=lfs -text
+approach/vlm/LLaVA/images/llava_example_cmp.png filter=lfs diff=lfs merge=lfs -text
+approach/vlm/LLaVA/images/llava_logo.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/.asset/ape.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/.asset/radar.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/.asset/demo.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/.asset/head.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/.asset/example_1.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/.asset/framework.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/demo/examples/SolvayConference1927.jpg filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/demo/examples/Terminator3.jpg filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/demo/examples/Totoro01.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/demo/examples/MatrixRevolutionForZion.jpg filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/demo/examples/Transformers.webp filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/ape/modeling/text/eva02_clip/bpe_simple_vocab_16e6.txt.gz filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/ape/modeling/text/eva01_clip/bpe_simple_vocab_16e6.txt.gz filter=lfs diff=lfs merge=lfs -text
+approach/vlm/LLaVA/images/llava_v1_5_radar.jpg filter=lfs diff=lfs merge=lfs -text
+approach/vlm/LLaVA/images/demo_cli.gif filter=lfs diff=lfs merge=lfs -text
+approach/ovod/mm-ovod/docs/teaser.jpg filter=lfs diff=lfs merge=lfs -text
+approach/ovod/mm-ovod/datasets/metadata/lvis_multi-modal_avg_K-005_author.npy filter=lfs diff=lfs merge=lfs -text
+approach/ovod/mm-ovod/datasets/metadata/lvis_image_exemplar_features_agg_K-005_author.npy filter=lfs diff=lfs merge=lfs -text
+approach/ovod/mm-ovod/datasets/metadata/lvis_gpt3_text-davinci-002_features_author.npy filter=lfs diff=lfs merge=lfs -text
+approach/ovod/mm-ovod/datasets/metadata/lvis_multi-modal_agg_K-005_author.npy filter=lfs diff=lfs merge=lfs -text
+approach/ovod/mm-ovod/datasets/metadata/lvis_image_exemplar_features_avg_K-005_author.npy filter=lfs diff=lfs merge=lfs -text
+approach/ovod/mm-ovod/datasets/metadata/lvis_v1_clip_a+cname.npy filter=lfs diff=lfs merge=lfs -text
+approach/ovod/d-cube/.assets/teaser.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/model_explan1.PNG filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/hero_figure.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/model_explan2.PNG filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/grounding_dino_logo.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/cats.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/cat_dog.jpeg filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/arch.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/COCO.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/GD_SD.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/ODinW.png filter=lfs diff=lfs merge=lfs -text
+approach/ovod/GroundingDINO/.asset/GD_GLIGEN.png filter=lfs diff=lfs merge=lfs -text
+baselines/Aria-UI/examples/aria.png filter=lfs diff=lfs merge=lfs -text
+baselines/Aria-UI/assets/overall.png filter=lfs diff=lfs merge=lfs -text
+baselines/Aria-UI/assets/logo_long.png filter=lfs diff=lfs merge=lfs -text
+baselines/Aria-UI/assets/aria_ui_framework_v4.pdf filter=lfs diff=lfs merge=lfs -text

approach/ovod/APE/ape/modeling/text/eva01_clip/bpe_simple_vocab_16e6.txt.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a
+size 1356917

approach/ovod/APE/ape/modeling/text/eva02_clip/bpe_simple_vocab_16e6.txt.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a
+size 1356917

approach/ovod/APE/ape_d_model_final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3548f41a3238148180e08fd4b16c71f4abc3ac3caf9c8434444462d1bdb7f965
+size 5956547279

approach/ovod/mm-ovod/datasets/metadata/lvis_gpt3_text-davinci-002_features_author.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48e39732dde55b2462222003714b2026b916cc982af2e4de43da64fa3e3a494f
+size 1232000

approach/ovod/mm-ovod/datasets/metadata/lvis_image_exemplar_features_agg_K-005_author.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4b83998ed03452204d17cf2f679d222b2338c973cf486784e88c9dcf8216a17
+size 1232000

approach/ovod/mm-ovod/datasets/metadata/lvis_image_exemplar_features_avg_K-005_author.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62a27438c4cb576080df9247ccabb2d6c6cd149244c3ebd054cfa88e169763f8
+size 2463872

approach/ovod/mm-ovod/datasets/metadata/lvis_multi-modal_agg_K-005_author.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dba2b82fd6f19930708c3feb1c0b0930c7c8f76ab00653105a96e9358afc5ae
+size 1232000

approach/ovod/mm-ovod/datasets/metadata/lvis_multi-modal_avg_K-005_author.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99ddedc9437a3a2173eb5a059c1e5b1e9e665778c744ee8dfa6f9d379011f6a9
+size 2463872

approach/ovod/mm-ovod/datasets/metadata/lvis_v1_clip_a+cname.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16059f1fe5f7d857273d16a1b7c6d6141a88d133e99a107be5065f42311da0f8
+size 1232000

baselines/Aria-UI/AndroidWorld/README.md

@@ -0,0 +1,57 @@
+# 🚀 Welcome to the M3A Agent Powered by Aria-UI for AndroidWorld!
+
+We’re thrilled to release the **M3A Agent powered by Aria-UI**! 🎉 This integration enhances task success rates and brings seamless grounding instruction understanding to **AndroidWorld**. Follow the steps below to get started. 🚀
+
+## 🛠️ How to Use M3A Agent with AndroidWorld
+
+### 1️⃣ Clone AndroidWorld and Set Up the Environment
+
+First, clone the latest version of **AndroidWorld** and install the required dependencies:
+
+```bash
+git clone https://github.com/google-research/android_world.git
+cd android_world
+```
+For more details, check out the official [AndroidWorld GitHub repository](https://github.com/google-research/android_world?tab=readme-ov-file).
+
+### 2️⃣ Merge M3A Agent Files with AndroidWorld
+
+We’ve organized our files to match AndroidWorld’s directory structure. To integrate:
+
+- Merge the provided `agents/` and `env/` directories with the respective directories in AndroidWorld.
+- Place `run_aria_ui.py` in the root directory of AndroidWorld.
+
+After merging, your directory structure should look like this:
+```
+AndroidWorld/
+├── agents/
+│   └── aria_ui_utils.py
+│   └── m3a_aria_ui.py
+├── env/
+│   └── json_action.py
+├── run_aria_ui.py
+...
+```
+
+### 3️⃣ Deploy Aria-UI API 🌐
+Under AndroidWorld/agents/aria_ui_utils.py, you’ll find how we connect to Aria-UI using an API. Follow these steps to set it up:
+- Deploy Aria-UI with vLLM to serve an OpenAI-style API.
+Reference the [vLLM OpenAI-Compatible Server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html).
+- Update the API key and base URL in `aria_ui_utils.py` to match your deployment. Replace the placeholders with your values:
+```python
+ariaui_api_key = "ariaui_api_key"
+ariaui_api_base = "ariaui_api_base"
+```
+
+**Note**: You may also set up a simple API server with the vLLM inference script and [FastAPI](https://github.com/fastapi/fastapi).
+### 4️⃣ Run the M3A Agent 🎉
+Once everything is set up, run the agent with the following command:
+```python
+python run_aria_ui.py
+```
+Sit back and watch **Aria-UI** in action! 🚀
+
+### 💡 Additional Notes
+For troubleshooting or further customization, explore `aria_ui_utils.py` and `m3a_aria_ui.py` to understand how Aria-UI is integrated.
+
+Enjoy using the M3A Agent powered by **Aria-UI**! 🎉

baselines/Aria-UI/AndroidWorld/agents/aria_ui_utils.py

@@ -0,0 +1,150 @@
+# Copyright 2024 The Aria-UI Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+from io import BytesIO
+import requests
+from PIL import Image
+from openai import OpenAI
+import numpy as np
+import cv2
+from android_world.agents.m3a_utils import _logical_to_physical
+
+"""
+Deploy Aria-UI with vLLM, then get the api_key and api_base from the deployment for directly API call.
+"""
+ariaui_api_key = "ariaui_api_key"
+ariaui_api_base = "ariaui_api_base"
+
+client = OpenAI(
+    api_key=ariaui_api_key,
+    base_url=ariaui_api_base,
+)
+
+models = client.models.list()
+model = models.data[0].id
+
+def encode_image_to_base64(image_path):
+    pil_image = Image.open(image_path).convert('RGB')
+    buffered = BytesIO()
+    pil_image.save(buffered, format="JPEG")
+    base64_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    return base64_str
+
+def encode_numpy_image_to_base64(image: np.ndarray) -> str:
+    """Converts a numpy array image to base64 string.
+    
+    Args:
+        image: Numpy array representing an image (height, width, channels)
+        
+    Returns:
+        Base64 encoded string of the image
+    """
+    # Convert numpy array to bytes
+    success, buffer = cv2.imencode('.jpg', image)
+    if not success:
+        raise ValueError("Failed to encode image to jpg format")
+    
+    # Convert bytes to base64 string
+    image_bytes = buffer.tobytes()
+    base64_string = base64.b64encode(image_bytes).decode('utf-8')
+    
+    return base64_string
+
+def request_aria_ui(image: np.ndarray, prompt: str) -> str:
+    image_base64 = encode_numpy_image_to_base64(image)
+    chat_completion_from_url = client.chat.completions.create(
+        messages=[{
+            "role":
+            "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": prompt
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{image_base64}"
+                    },
+                },
+            ],
+        }],
+        model=model,
+        max_tokens=512,
+        stop=["<|im_end|>"],
+        extra_body= {
+            "split_image": True,
+            "image_max_size": 980
+        }
+    )
+
+    result = chat_completion_from_url.choices[0].message.content
+    print(f"Chat completion output:{result}")
+    return result
+
+
+def add_ui_element_mark_coords(
+    screenshot: np.ndarray,
+    coords: tuple[int, int],  # Normalized coordinates in [0, 1000]
+    logical_screen_size: tuple[int, int],
+    physical_frame_boundary: tuple[int, int, int, int],
+    orientation: int,
+):
+    """Add a red circle marker at the specified normalized coordinates.
+
+    Args:
+        screenshot: The screenshot as a numpy ndarray.
+        coords: Normalized coordinates (x, y) in range [0, 1000].
+        logical_screen_size: The logical screen size.
+        physical_frame_boundary: The physical coordinates in portrait orientation
+          for the upper left and lower right corner for the frame.
+        orientation: The current screen orientation.
+    """
+    # Convert normalized coordinates to logical coordinates
+    logical_point = (
+        coords[0] * logical_screen_size[0] // 1000,
+        coords[1] * logical_screen_size[1] // 1000
+    )
+    
+    # Convert to physical coordinates
+    physical_point = _logical_to_physical(
+        logical_point,
+        logical_screen_size,
+        physical_frame_boundary,
+        orientation,
+    )
+
+    # Draw a large red circle
+    radius = 30  # Adjust size as needed
+    cv2.circle(
+        screenshot,
+        physical_point,
+        radius,
+        color=(0, 0, 255),  # BGR format - Red
+        thickness=3
+    )
+
+def convert_coords_to_physical(coords: tuple[int, int], logical_screen_size: tuple[int, int], physical_frame_boundary: tuple[int, int, int, int], orientation: int) -> tuple[int, int]:
+    logical_point = (
+        coords[0] * logical_screen_size[0] // 1000,
+        coords[1] * logical_screen_size[1] // 1000
+    )
+    physical_point = _logical_to_physical(
+        logical_point,
+        logical_screen_size,
+        physical_frame_boundary,
+        orientation,
+    )
+    return physical_point

baselines/Aria-UI/AndroidWorld/agents/m3a_aria_ui.py

@@ -0,0 +1,668 @@
+# Copyright 2024 The Aria-UI Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright 2024 The android_world Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A Multimodal Autonomous Agent for Android (Aria-UI under M3A)."""
+
+import time
+from android_world.agents import agent_utils
+from android_world.agents import base_agent
+from android_world.agents import infer
+from android_world.agents import m3a_utils
+from android_world.env import interface
+from android_world.env import json_action
+from android_world.env import representation_utils
+from android_world.agents import aria_ui_utils
+import ast
+from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type
+
+
+PROMPT_PREFIX = """
+You are an agent who can operate an Android phone on behalf of a user. Based on the user's goal/request, you may:
+
+- Answer back if the request/goal is a question (or a chat message), like user asks "What is my schedule for today?".  
+- Complete some tasks described in the requests/goals by performing actions (step by step) on the phone.
+
+When given a user request, you will try to complete it step by step. At each step, you will be given the current screenshot and a history of what you have done (in text). Based on these pieces of information and the goal, you must choose to perform one of the actions in the following list (action description followed by the JSON format) by outputting the action in the correct JSON format:
+
+- If you think the task has been completed, finish the task by using the status action with complete as goal_status:  
+  {{
+    "action_type": "status", 
+    "goal_status": "complete"
+  }}
+
+- If you think the task is not feasible (including cases like you don't have enough information or cannot perform some necessary actions), finish by using the `status` action with infeasible as goal_status:  
+  {{
+    "action_type": "status", 
+    "goal_status": "infeasible"
+  }}
+
+- Answer user's question:  
+  {{
+    "action_type": "answer", 
+    "text": "answer_text"
+  }}
+
+- Click/tap on an element on the screen. Please describe the element you want to click using natural language:  
+  {{
+    "action_type": "click", 
+    "instruction": the step-wise instruction in short,
+    "target": target_element_description
+  }}
+
+- Long press on an element on the screen, similar to the click action above, use the semantic description to indicate the element:  
+  {{
+    "action_type": "long_press", 
+    "instruction": the step-wise instruction in short,
+    "target": target_element_description
+  }}
+
+- Type text into a text field (this action contains clicking the text field, typing in the text, and pressing Enter, so no need to click on the target field to start). Use the semantic description to indicate the target text field:  
+  {{
+    "action_type": "input_text", 
+    "text": text_input, 
+    "instruction": the step-wise instruction in short,
+    "target": target_element_description
+  }}
+
+- Press the Enter key:  
+  {{
+    "action_type": "keyboard_enter"
+  }}
+
+- Navigate to the home screen:  
+  {{
+    "action_type": "navigate_home"
+  }}
+
+- Navigate back:  
+  {{
+    "action_type": "navigate_back"
+  }}
+
+- Scroll the screen or a scrollable UI element in one of the four directions, use the same semantic description as above if you want to scroll a specific UI element, leave it empty when scrolling the whole screen:  
+  {{
+    "action_type": "scroll", 
+    "direction": "up/down/left/right", 
+    "instruction": the step-wise instruction in short,
+    "element": optional_target_element_description
+  }}
+
+- Open an app (nothing will happen if the app is not installed):  
+  {{
+    "action_type": "open_app", 
+    "app_name": name
+  }}
+
+- Wait for the screen to update:  
+  {{
+    "action_type": "wait"
+  }}
+"""
+
+GUIDANCE = """
+Here are some useful guidelines you need to follow:
+
+General:
+- Usually there will be multiple ways to complete a task, pick the easiest one. Also when something does not work as expected (due to various reasons), sometimes a simple retry can solve the problem, but if it doesn’t (you can see that from the history), SWITCH to other solutions.
+- Sometimes you may need to navigate the phone to gather information needed to complete the task, for example if user asks "what is my schedule tomorrow", then you may want to open the calendar app (using the ‘open_app‘ action), look up information there, answer user’s question (using the ‘answer‘ action) and finish (using the ‘status‘ action with complete as goal_status).
+- For requests that are questions (or chat messages), remember to use the ‘answer‘ action to reply to user explicitly before finish! Merely displaying the answer on the screen is NOT sufficient (unless the goal is something like "show me ...").
+- If the desired state is already achieved (e.g., enabling Wi-Fi when it’s already on), you can just complete the task.
+
+Action Related:
+- Use the ‘open_app‘ action whenever you want to open an app (nothing will happen if the app is not installed), do not use the app drawer to open an app unless all other ways have failed.
+- Use the ‘input_text‘ action whenever you want to type something (including passwords) instead of clicking characters on the keyboard one by one. Sometimes there is some default text in the text field you want to type in, remember to delete them before typing.
+- For ‘click‘, ‘long_press‘ and ‘input_text‘, the target_element.description parameter you choose must be based on a VISIBLE element in the screenshot.
+- Consider exploring the screen by using the ‘scroll‘ action with different directions to reveal additional content.
+- The direction parameter for the ‘scroll‘ action can be confusing sometimes as it’s opposite to swipe, for example, to view content at the bottom, the ‘scroll‘ direction should be set to "down". It has been observed that you have difficulties in choosing the correct direction, so if one does not work, try the opposite as well.
+
+Text Related Operations:
+- Normally to select certain text on the screen: (i) Enter text selection mode by long pressing the area where the text is, then some of the words near the long press point will be selected (highlighted with two pointers indicating the range) and usually a text selection bar will also appear with options like "copy", "paste", "select all", etc. (ii) Select the exact text you need. Usually the text selected from the previous step is NOT the one you want, you need to adjust the range by dragging the two pointers. If you want to select all text in the text field, simply click the "select all" button in the bar.
+- At this point, you don’t have the ability to drag something around the screen, so in general you can not select arbitrary text.
+- To delete some text: the most traditional way is to place the cursor at the right place and use the backspace button in the keyboard to delete the characters one by one (can long press the backspace to accelerate if there are many to delete). Another approach is to first select the text you want to delete, then click the backspace button in the keyboard.
+- To copy some text: first select the exact text you want to copy, which usually also brings up the text selection bar, then click the `copy` button in bar.
+- To paste text into a text box, first long press the text box, then usually the text selection bar will appear with a `paste` button in it.
+- When typing into a text field, sometimes an auto-complete dropdown list will appear. This usually indicating this is a enum field and you should try to select the best match by clicking the corresponding one in the list.
+"""
+
+
+ACTION_SELECTION_PROMPT_TEMPLATE = (
+    PROMPT_PREFIX + "\nThe current user goal/request is: {goal}\n\n"
+    "Here is a history of what you have done so far:\n{history}\n\n"
+    "The current screenshot is also given to you.\n"
+    + GUIDANCE
+    + "{additional_guidelines}"
+    + "\nNow output an action from the above list in the correct JSON format,"
+    " following the reason why you do that. Your answer should look like:\n"
+    'Reason: ...\nAction: {{"action_type":...}}\n\n'
+    "Your Answer:\n"
+)
+
+
+SUMMARY_PROMPT_TEMPLATE = (
+    PROMPT_PREFIX + "\nThe (overall) user goal/request is: {goal}\n"
+    "Now I want you to summerize the latest step.\n"
+    "You will be given the screenshot before you performed the action (which"
+    ' has a text label "before" on the bottom right), the action you chose'
+    " (together with the reason) and the screenshot after the action was"
+    ' performed (which has a text label "after" on the bottom right).\n'
+    "Also here is the list of detailed information for some UI elements"
+    " in the before screenshot:\n{before_elements}\n"
+    "Here is the list for the after screenshot:\n{after_elements}\n"
+    "This is the action you picked: {action}\n"
+    "Based on the reason: {reason}\n\n"
+    "By comparing the two screenshots (plus the UI element lists) and the"
+    " action performed, give a brief summary of this step. This summary"
+    " will be added to action history and used in future action selection,"
+    " so try to include essential information you think that will be most"
+    " useful for future action selections like what you"
+    " intended to do, why, if it worked as expected, if not"
+    " what might be the reason (be critical, the action/reason might be"
+    " wrong), what should/should not be done next and so on. Some more"
+    " rules/tips you should follow:\n"
+    "- Keep it short (better less than 50 words) and in a single line\n"
+    "- Some actions (like `answer`, `wait`) don't involve screen change,"
+    " you can just assume they work as expected.\n"
+    "- Given this summary will be added into action history, it can be used as"
+    " memory to include information that needs to be remembered, or shared"
+    " between different apps.\n\n"
+    "Summary of this step: "
+)
+
+
+ARIA_UI_PROMPT_TEMPLATE = """The agent is performing the ultimate task: {ultimate_task}.
+History of the agent's steps:\n{history_list}.
+<image>Step {step_idx}. Given a GUI image, what are the relative (0-1000) pixel point coordinates for the element corresponding to the following instruction or description: {instruction}"""
+
+ARIA_UI_PROMPT_TEMPLATE_MINIWOB = """<image>Given a GUI image, what are the relative (0-1000) pixel point coordinates for the element corresponding to the following instruction or description: {instruction}"""
+
+
+def _generate_ui_element_description(
+    ui_element: representation_utils.UIElement, index: int
+) -> str:
+    """Generate a description for a given UI element with important information.
+
+    Args:
+      ui_element: UI elements for the current screen.
+      index: The numeric index for the UI element.
+
+    Returns:
+      The description for the UI element.
+    """
+    element_description = f'UI element {index}: {{"index": {index}, '
+    if ui_element.text:
+        element_description += f'"text": "{ui_element.text}", '
+    if ui_element.content_description:
+        element_description += (
+            f'"content_description": "{ui_element.content_description}", '
+        )
+    if ui_element.hint_text:
+        element_description += f'"hint_text": "{ui_element.hint_text}", '
+    if ui_element.tooltip:
+        element_description += f'"tooltip": "{ui_element.tooltip}", '
+    element_description += (
+        f'"is_clickable": {"True" if ui_element.is_clickable else "False"}, '
+    )
+    element_description += (
+        '"is_long_clickable":'
+        f' {"True" if ui_element.is_long_clickable else "False"}, '
+    )
+    element_description += (
+        f'"is_editable": {"True" if ui_element.is_editable else "False"}, '
+    )
+    if ui_element.is_scrollable:
+        element_description += '"is_scrollable": True, '
+    if ui_element.is_focusable:
+        element_description += '"is_focusable": True, '
+    element_description += (
+        f'"is_selected": {"True" if ui_element.is_selected else "False"}, '
+    )
+    element_description += (
+        f'"is_checked": {"True" if ui_element.is_checked else "False"}, '
+    )
+    return element_description[:-2] + "}"
+
+
+def _generate_ui_elements_description_list(
+    ui_elements: list[representation_utils.UIElement],
+    screen_width_height_px: tuple[int, int],
+) -> str:
+    """Generate concise information for a list of UIElement.
+
+    Args:
+      ui_elements: UI elements for the current screen.
+      screen_width_height_px: The height and width of the screen in pixels.
+
+    Returns:
+      Concise information for each UIElement.
+    """
+    tree_info = ""
+    for index, ui_element in enumerate(ui_elements):
+        if m3a_utils.validate_ui_element(ui_element, screen_width_height_px):
+            tree_info += _generate_ui_element_description(ui_element, index) + "\n"
+    return tree_info
+
+
+def _pvision_action_selection_prompt(
+    goal: str,
+    history: list[str],
+    additional_guidelines: list[str] | None = None,
+) -> str:
+    """Generate the prompt for the action selection.
+
+    Args:
+      goal: The current goal.
+      history: Summaries for previous steps.
+      ui_elements: A list of descriptions for the UI elements.
+      additional_guidelines: Task specific guidelines.
+
+    Returns:
+      The text prompt for action selection that will be sent to gpt4v.
+    """
+    if history:
+        history = "\n".join(history)
+    else:
+        history = "You just started, no action has been performed yet."
+
+    extra_guidelines = ""
+    if additional_guidelines:
+        extra_guidelines = "For The Current Task:\n"
+        for guideline in additional_guidelines:
+            extra_guidelines += f"- {guideline}\n"
+
+    return ACTION_SELECTION_PROMPT_TEMPLATE.format(
+        goal=goal,
+        history=history,
+        additional_guidelines=extra_guidelines,
+    )
+
+
+def _summarize_prompt(
+    action: str,
+    reason: str,
+    goal: str,
+    before_elements: str,
+    after_elements: str,
+) -> str:
+    """Generate the prompt for the summarization step.
+
+    Args:
+      action: Action picked.
+      reason: The reason to pick the action.
+      goal: The overall goal.
+      before_elements: Information for UI elements on the before screenshot.
+      after_elements: Information for UI elements on the after screenshot.
+
+    Returns:
+      The text prompt for summarization that will be sent to gpt4v.
+    """
+    return SUMMARY_PROMPT_TEMPLATE.format(
+        goal=goal,
+        before_elements=before_elements,
+        after_elements=after_elements,
+        action=action,
+        reason=reason,
+    )
+
+
+def _extract_coords_from_response(response: str) -> tuple[int, int]:
+    """Extract coordinate tuple from LLM response string.
+
+    Args:
+        response: String containing coordinates like "(892,925)" or "[892, 925]"
+
+    Returns:
+        Tuple of (x,y) coordinates as integers
+
+    Raises:
+        ValueError: If exactly 2 numbers are not found in the response
+    """
+    # Clean up the response string
+    resp = response.replace("```", "").strip()
+
+    # Extract numbers using regex
+    import re
+
+    numbers = re.findall(r"\d+", resp)
+    if len(numbers) != 2:
+        raise ValueError(
+            f"Expected exactly 2 coordinates, found {len(numbers)} numbers in response: {response}"
+        )
+
+    return (int(numbers[0]), int(numbers[1]))
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_fixed(5),
+    retry=retry_if_exception_type(ValueError),
+    reraise=True,
+)
+def call_grounding_llm(screenshot, goal, history, elem_description, elem_instruction):
+    history_list = "\n".join(
+        [f"\t{j+1}. " + step_info["summary"] for j, step_info in enumerate(history)]
+    )
+    """
+    AndroidWorld
+    """
+    prompt = ARIA_UI_PROMPT_TEMPLATE.format(
+        ultimate_task=goal,
+        history_list=history_list,
+        step_idx=len(history) + 1,
+        instruction=f"description: {elem_description}; instruction: {elem_instruction}",
+    )
+    """
+    MiniWob++
+    """
+    # prompt = ARIA_UI_PROMPT_TEMPLATE_MINIWOB.format(
+    #     instruction=f"{elem_description}",
+    # )
+
+    response = aria_ui_utils.request_aria_ui(screenshot, prompt)
+
+    coords = _extract_coords_from_response(response)
+    return coords
+
+
+class M3A(base_agent.EnvironmentInteractingAgent):
+    """M3A which stands for Multimodal Autonomous Agent for Android."""
+
+    def __init__(
+        self,
+        env: interface.AsyncEnv,
+        llm: infer.MultimodalLlmWrapper,
+        name: str = "M3A",
+        wait_after_action_seconds: float = 2.0,
+    ):
+        """Initializes a M3A Agent.
+
+        Args:
+          env: The environment.
+          llm: The multimodal LLM wrapper.
+          name: The agent name.
+          wait_after_action_seconds: Seconds to wait for the screen to stablize
+            after executing an action
+        """
+        super().__init__(env, name)
+        self.llm = llm
+        self.history = []
+        self.additional_guidelines = None
+        self.wait_after_action_seconds = wait_after_action_seconds
+
+    def set_task_guidelines(self, task_guidelines: list[str]) -> None:
+        self.additional_guidelines = task_guidelines
+
+    def reset(self, go_home_on_reset: bool = False):
+        super().reset(go_home_on_reset)
+        # Hide the coordinates on screen which might affect the vision model.
+        self.env.hide_automation_ui()
+        self.history = []
+
+    def step(self, goal: str) -> base_agent.AgentInteractionResult:
+        step_data = {
+            "raw_screenshot": None,
+            "before_screenshot_with_som": None,
+            "before_ui_elements": [],
+            "after_screenshot_with_som": None,
+            "action_prompt": None,
+            "action_output": None,
+            "action_output_json": None,
+            "action_reason": None,
+            "action_raw_response": None,
+            "summary_prompt": None,
+            "summary": None,
+            "summary_raw_response": None,
+        }
+        print("----------step " + str(len(self.history) + 1))
+
+        state = self.get_post_transition_state()
+        logical_screen_size = self.env.logical_screen_size
+        orientation = self.env.orientation
+        physical_frame_boundary = self.env.physical_frame_boundary
+
+        before_ui_elements = state.ui_elements
+        step_data["before_ui_elements"] = before_ui_elements
+        before_ui_elements_list = _generate_ui_elements_description_list(
+            before_ui_elements, logical_screen_size
+        )
+        step_data["raw_screenshot"] = state.pixels.copy()
+        before_screenshot = state.pixels.copy()
+        for index, ui_element in enumerate(before_ui_elements):
+            if m3a_utils.validate_ui_element(ui_element, logical_screen_size):
+                m3a_utils.add_ui_element_mark(
+                    before_screenshot,
+                    ui_element,
+                    index,
+                    logical_screen_size,
+                    physical_frame_boundary,
+                    orientation,
+                )
+        step_data["before_screenshot_with_som"] = before_screenshot.copy()
+
+        action_prompt = _pvision_action_selection_prompt(
+            goal,
+            [
+                "Step " + str(i + 1) + "- " + step_info["summary"]
+                for i, step_info in enumerate(self.history)
+            ],
+            self.additional_guidelines,
+        )
+        step_data["action_prompt"] = action_prompt
+        action_output, is_safe, raw_response = self.llm.predict_mm(
+            action_prompt,
+            [
+                step_data["raw_screenshot"],
+            ],
+        )
+
+        if is_safe == False:  # pylint: disable=singleton-comparison
+            #  is_safe could be None
+            action_output = f"""Reason: {m3a_utils.TRIGGER_SAFETY_CLASSIFIER}
+Action: {{"action_type": "status", "goal_status": "infeasible"}}"""
+
+        if not raw_response:
+            raise RuntimeError("Error calling LLM in action selection phase.")
+        step_data["action_output"] = action_output
+        step_data["action_raw_response"] = raw_response
+
+        reason, action = m3a_utils.parse_reason_action_output(action_output)
+
+        # If the output is not in the right format, add it to step summary which
+        # will be passed to next step and return.
+        if (not reason) or (not action):
+            print("Action prompt output is not in the correct format.")
+            step_data["summary"] = (
+                "Output for action selection is not in the correct format, so no"
+                " action is performed."
+            )
+            self.history.append(step_data)
+
+            return base_agent.AgentInteractionResult(
+                False,
+                step_data,
+            )
+
+        print("Action: " + action)
+        print("Reason: " + reason)
+        step_data["action_reason"] = reason
+
+        try:
+            action_json = agent_utils.extract_json(action)
+            converted_action = json_action.JSONAction(
+                **action_json,
+            )
+            step_data["action_output_json"] = converted_action
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            print("Failed to convert the output to a valid action.")
+            print(str(e))
+            step_data["summary"] = (
+                "Can not parse the output to a valid action. Please make sure to pick"
+                " the action from the list with required parameters (if any) in the"
+                " correct JSON format!"
+            )
+            self.history.append(step_data)
+
+            return base_agent.AgentInteractionResult(
+                False,
+                step_data,
+            )
+
+        action_index = converted_action.index
+        elem_description = converted_action.target
+        elem_instruction = converted_action.instruction
+
+        num_ui_elements = len(before_ui_elements)
+        if (
+            converted_action.action_type
+            in ["click", "long_press", "input_text", "scroll"]
+            and elem_description is not None
+        ):
+            aria_ui_coords = call_grounding_llm(
+                step_data["raw_screenshot"],
+                goal,
+                self.history,
+                elem_description,
+                elem_instruction,
+            )
+            physical_coords = aria_ui_utils.convert_coords_to_physical(
+                aria_ui_coords,
+                logical_screen_size,
+                physical_frame_boundary,
+                orientation,
+            )
+
+            converted_action.x = physical_coords[0]
+            converted_action.y = physical_coords[1]
+
+            # Add mark to the target element.
+            aria_ui_utils.add_ui_element_mark_coords(
+                step_data["raw_screenshot"],
+                aria_ui_coords,
+                logical_screen_size,
+                physical_frame_boundary,
+                orientation,
+            )
+
+        if converted_action.action_type == "status":
+            if converted_action.goal_status == "infeasible":
+                print("Agent stopped since it thinks mission impossible.")
+            step_data["summary"] = "Agent thinks the request has been completed."
+            self.history.append(step_data)
+            return base_agent.AgentInteractionResult(
+                True,
+                step_data,
+            )
+
+        if converted_action.action_type == "answer":
+            print("Agent answered with: " + converted_action.text)
+
+        try:
+            self.env.execute_action(converted_action)
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            print("Failed to execute action.")
+            print(str(e))
+            step_data["summary"] = (
+                "Can not execute the action, make sure to select the action with"
+                " the required parameters (if any) in the correct JSON format!"
+            )
+            return base_agent.AgentInteractionResult(
+                False,
+                step_data,
+            )
+
+        time.sleep(self.wait_after_action_seconds)
+
+        state = self.env.get_state(wait_to_stabilize=False)
+        logical_screen_size = self.env.logical_screen_size
+        orientation = self.env.orientation
+        physical_frame_boundary = self.env.physical_frame_boundary
+        after_ui_elements = state.ui_elements
+        after_ui_elements_list = _generate_ui_elements_description_list(
+            after_ui_elements, logical_screen_size
+        )
+        after_screenshot = state.pixels.copy()
+        for index, ui_element in enumerate(after_ui_elements):
+            if m3a_utils.validate_ui_element(ui_element, logical_screen_size):
+                m3a_utils.add_ui_element_mark(
+                    after_screenshot,
+                    ui_element,
+                    index,
+                    logical_screen_size,
+                    physical_frame_boundary,
+                    orientation,
+                )
+
+        m3a_utils.add_screenshot_label(
+            step_data["before_screenshot_with_som"], "before"
+        )
+        m3a_utils.add_screenshot_label(after_screenshot, "after")
+        step_data["after_screenshot_with_som"] = after_screenshot.copy()
+
+        summary_prompt = _summarize_prompt(
+            action,
+            reason,
+            goal,
+            before_ui_elements_list,
+            after_ui_elements_list,
+        )
+        summary, is_safe, raw_response = self.llm.predict_mm(
+            summary_prompt,
+            [
+                before_screenshot,
+                after_screenshot,
+            ],
+        )
+
+        if is_safe == False:  # pylint: disable=singleton-comparison
+            #  is_safe could be None
+            summary = """Summary triggered LLM safety classifier."""
+
+        if not raw_response:
+            print(
+                "Error calling LLM in summarization phase. This should not happen: "
+                f"{summary}"
+            )
+            step_data["summary"] = (
+                "Some error occurred calling LLM during summarization phase: %s"
+                % summary
+            )
+            self.history.append(step_data)
+            return base_agent.AgentInteractionResult(
+                False,
+                step_data,
+            )
+
+        step_data["summary_prompt"] = summary_prompt
+        step_data["summary"] = f"Action selected: {action}. {summary}"
+        print("Summary: " + summary)
+        step_data["summary_raw_response"] = raw_response
+
+        self.history.append(step_data)
+        return base_agent.AgentInteractionResult(
+            False,
+            step_data,
+        )

baselines/Aria-UI/AndroidWorld/env/json_action.py

@@ -0,0 +1,180 @@
+# Copyright 2024 The android_world Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Represents an action for Android interaction, parsed from a JSON format."""
+
+import dataclasses
+import json
+from typing import Optional
+
+
+_JSON_SEPARATORS = (',', ':')
+
+ANSWER = 'answer'
+CLICK = 'click'
+DOUBLE_TAP = 'double_tap'
+INPUT_TEXT = 'input_text'
+KEYBOARD_ENTER = 'keyboard_enter'
+LONG_PRESS = 'long_press'
+NAVIGATE_BACK = 'navigate_back'
+NAVIGATE_HOME = 'navigate_home'
+OPEN_APP = 'open_app'
+SCROLL = 'scroll'
+STATUS = 'status'
+SWIPE = 'swipe'
+UNKNOWN = 'unknown'
+WAIT = 'wait'
+
+_ACTION_TYPES = (
+    CLICK,
+    DOUBLE_TAP,
+    SCROLL,
+    SWIPE,
+    INPUT_TEXT,
+    NAVIGATE_HOME,
+    NAVIGATE_BACK,
+    KEYBOARD_ENTER,
+    OPEN_APP,
+    STATUS,
+    WAIT,
+    LONG_PRESS,
+    ANSWER,
+    UNKNOWN,
+)
+
+_SCROLL_DIRECTIONS = ('left', 'right', 'down', 'up')
+
+# Keys of JSON action.
+ACTION_TYPE = 'action_type'
+INDEX = 'index'
+X = 'x'
+Y = 'y'
+TEXT = 'text'
+DIRECTION = 'direction'
+APP_NAME = 'app_name'
+GOAL_STATUS = 'goal_status'
+
+
+@dataclasses.dataclass()
+class JSONAction:
+  """Represents a parsed JSON action.
+
+  # Example
+  result_json = {'action_type': 'click', 'x': %d, 'y': %d}
+  action = JSONAction(**result_json)
+
+  Attributes:
+    action_type: The action type.
+    index: The index to click, if action is a click. Either an index or a <x, y>
+      should be provided. See x, y attributes below.
+    x: The x position to click, if the action is a click.
+    y: The y position to click, if the action is a click.
+    text: The text to type, if action is type.
+    direction: The direction to scroll, if action is scroll.
+    goal_status: If the status is a 'status' type, indicates the status of the
+      goal.
+    app_name: The app name to launch, if the action type is 'open_app'.
+    keycode: Keycode actions are necessary for an agent to interact with complex
+      UI elements (like large textareas) that can't be accessed or controlled by
+      simply taping, ensuring precise control over navigation and selection in
+      the interface.
+  """
+
+  action_type: Optional[str] = None
+  index: Optional[str | int] = None
+  x: Optional[int] = None
+  y: Optional[int] = None
+  text: Optional[str] = None
+  direction: Optional[str] = None
+  goal_status: Optional[str] = None
+  app_name: Optional[str] = None
+  keycode: Optional[str] = None
+
+  # for aria-UI
+  target: Optional[str] = None
+  instruction: Optional[str] = None
+  coords: Optional[tuple[int, int]] = None
+
+  def __post_init__(self):
+    if self.action_type not in _ACTION_TYPES:
+      raise ValueError(f'Invalid action type: {self.action_type}')
+    if self.index is not None:
+      self.index = int(self.index)
+      if self.x is not None or self.y is not None:
+        raise ValueError('Either an index or a <x, y> should be provided.')
+    if self.direction and self.direction not in _SCROLL_DIRECTIONS:
+      raise ValueError(f'Invalid scroll direction: {self.direction}')
+    if self.text is not None and not isinstance(self.text, str):
+      self.text = str(self.text)
+    if self.keycode is not None and not self.keycode.startswith('KEYCODE_'):
+      raise ValueError(f'Invalid keycode: {self.keycode}')
+
+  def __repr__(self) -> str:
+    properties = []
+    for key, value in self.__dict__.items():
+      if value is not None:
+        if isinstance(value, float):
+          value = f'{value:.3f}'
+        properties.append(f'{key}={value!r}')
+    return f"JSONAction({', '.join(properties)})"
+
+  def __eq__(self, other):
+    if isinstance(other, JSONAction):
+      return _compare_actions(self, other)
+    return False
+
+  def __ne__(self, other):
+    return not self.__eq__(other)
+
+  def json_str(self) -> str:
+    non_null = {}
+    for key, value in self.__dict__.items():
+      if value is not None:
+        non_null[key] = value
+    return json.dumps(non_null, separators=_JSON_SEPARATORS)
+
+
+def _compare_actions(a: JSONAction, b: JSONAction) -> bool:
+  """Compares two JSONActions.
+
+  Args:
+    a: The first action.
+    b: The second action.
+
+  Returns:
+    If the actions are equal.
+  """
+  # Ignore cases.
+  if a.app_name is not None and b.app_name is not None:
+    app_name_match = a.app_name.lower() == b.app_name.lower()
+  else:
+    app_name_match = a.app_name == b.app_name
+
+  if a.text is not None and b.text is not None:
+    text_match = a.text.lower() == b.text.lower()
+  else:
+    text_match = a.text == b.text
+
+  # Compare the non-metadata fields.
+  return (
+      app_name_match
+      and text_match
+      and a.action_type == b.action_type
+      and a.index == b.index
+      and a.x == b.x
+      and a.y == b.y
+      and a.keycode == b.keycode
+      and a.direction == b.direction
+      and a.goal_status == b.goal_status
+  )

baselines/Aria-UI/AndroidWorld/run_aria_ui.py

@@ -0,0 +1,245 @@
+# Copyright 2024 The Aria-UI Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright 2024 The android_world Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Run eval suite.
+
+The run.py module is used to run a suite of tasks, with configurable task
+combinations, environment setups, and agent configurations. You can run specific
+tasks or all tasks in the suite and customize various settings using the
+command-line flags.
+"""
+
+from collections.abc import Sequence
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+from android_world import checkpointer as checkpointer_lib
+from android_world import registry
+from android_world import suite_utils
+from android_world.agents import base_agent
+from android_world.agents import human_agent
+from android_world.agents import infer
+from android_world.agents import m3a
+from android_world.agents import random_agent
+from android_world.agents import seeact
+from android_world.agents import t3a
+from android_world.env import env_launcher
+from android_world.env import interface
+
+from android_world.agents import m3a_aria_ui
+
+logging.set_verbosity(logging.WARNING)
+
+os.environ["GRPC_VERBOSITY"] = "ERROR"  # Only show errors
+os.environ["GRPC_TRACE"] = "none"  # Disable tracing
+
+
+def _find_adb_directory() -> str:
+    """Returns the directory where adb is located."""
+    potential_paths = [
+        os.path.expanduser("~/Library/Android/sdk/platform-tools/adb"),
+        os.path.expanduser("~/Android/Sdk/platform-tools/adb"),
+    ]
+    for path in potential_paths:
+        if os.path.isfile(path):
+            return path
+    raise EnvironmentError(
+        "adb not found in the common Android SDK paths. Please install Android"
+        " SDK and ensure adb is in one of the expected directories. If it's"
+        " already installed, point to the installed location."
+    )
+
+
+_ADB_PATH = flags.DEFINE_string(
+    "adb_path",
+    _find_adb_directory(),
+    "Path to adb. Set if not installed through SDK.",
+)
+_EMULATOR_SETUP = flags.DEFINE_boolean(
+    "perform_emulator_setup",
+    False,
+    "Whether to perform emulator setup. This must be done once and only once"
+    " before running Android World. After an emulator is setup, this flag"
+    " should always be False.",
+)
+_DEVICE_CONSOLE_PORT = flags.DEFINE_integer(
+    "console_port",
+    5554,
+    "The console port of the running Android device. This can usually be"
+    " retrieved by looking at the output of `adb devices`. In general, the"
+    " first connected device is port 5554, the second is 5556, and"
+    " so on.",
+)
+
+_SUITE_FAMILY = flags.DEFINE_enum(
+    "suite_family",
+    registry.TaskRegistry.ANDROID_WORLD_FAMILY,
+    [
+        # Families from the paper.
+        registry.TaskRegistry.ANDROID_WORLD_FAMILY,
+        registry.TaskRegistry.MINIWOB_FAMILY_SUBSET,
+        # Other families for more testing.
+        registry.TaskRegistry.MINIWOB_FAMILY,
+        registry.TaskRegistry.ANDROID_FAMILY,
+        registry.TaskRegistry.INFORMATION_RETRIEVAL_FAMILY,
+    ],
+    "Suite family to run. See registry.py for more information.",
+)
+_TASK_RANDOM_SEED = flags.DEFINE_integer(
+    "task_random_seed", 30, "Random seed for task randomness."
+)
+
+_TASKS = flags.DEFINE_list(
+    "tasks",
+    None,
+    "List of specific tasks to run in the given suite family. If None, run all"
+    " tasks in the suite family.",
+)
+_N_TASK_COMBINATIONS = flags.DEFINE_integer(
+    "n_task_combinations",
+    1,
+    "Number of task instances to run for each task template.",
+)
+
+_CHECKPOINT_DIR = flags.DEFINE_string(
+    "checkpoint_dir",
+    "",
+    "The directory to save checkpoints and resume evaluation from. If the"
+    " directory contains existing checkpoint files, evaluation will resume from"
+    " the latest checkpoint. If the directory is empty or does not exist, a new"
+    " directory will be created.",
+)
+_OUTPUT_PATH = flags.DEFINE_string(
+    "output_path",
+    os.path.expanduser("~/android_world/runs"),
+    "The path to save results to if not resuming from a checkpoint is not" " provided.",
+)
+
+# Agent specific.
+_AGENT_NAME = flags.DEFINE_string("agent_name", "m3a_aria_ui", help="Agent name.")
+
+_FIXED_TASK_SEED = flags.DEFINE_boolean(
+    "fixed_task_seed",
+    False,
+    "Whether to use the same task seed when running multiple task combinations"
+    " (n_task_combinations > 1).",
+)
+
+
+# MiniWoB is very lightweight and new screens/View Hierarchy load quickly.
+_MINIWOB_TRANSITION_PAUSE = 0.2
+
+# Additional guidelines for the MiniWob tasks.
+_MINIWOB_ADDITIONAL_GUIDELINES = [
+    (
+        "This task is running in a mock app, you must stay in this app and"
+        " DO NOT use the `navigate_home` action."
+    ),
+]
+
+
+def _get_agent(
+    env: interface.AsyncEnv,
+    family: str | None = None,
+) -> base_agent.EnvironmentInteractingAgent:
+    """Gets agent."""
+    print("Initializing agent...")
+    agent = None
+
+    agent = m3a_aria_ui.M3A(env, infer.Gpt4Wrapper("gpt-4o"))
+
+    if (
+        agent.name in ["M3A", "T3A", "SeeAct"]
+        and family
+        and family.startswith("miniwob")
+        and hasattr(agent, "set_task_guidelines")
+    ):
+        agent.set_task_guidelines(_MINIWOB_ADDITIONAL_GUIDELINES)
+    agent.name = _AGENT_NAME.value
+
+    return agent
+
+
+def _main() -> None:
+    """Runs eval suite and gets rewards back."""
+    env = env_launcher.load_and_setup_env(
+        console_port=_DEVICE_CONSOLE_PORT.value,
+        emulator_setup=_EMULATOR_SETUP.value,
+        adb_path=_ADB_PATH.value,
+    )
+
+    n_task_combinations = _N_TASK_COMBINATIONS.value
+    task_registry = registry.TaskRegistry()
+    suite = suite_utils.create_suite(
+        task_registry.get_registry(family=_SUITE_FAMILY.value),
+        n_task_combinations=n_task_combinations,
+        seed=_TASK_RANDOM_SEED.value,
+        tasks=_TASKS.value,
+        use_identical_params=_FIXED_TASK_SEED.value,
+    )
+    suite.suite_family = _SUITE_FAMILY.value
+
+    agent = _get_agent(env, _SUITE_FAMILY.value)
+
+    if _SUITE_FAMILY.value.startswith("miniwob"):
+        # MiniWoB pages change quickly, don't need to wait for screen to stabilize.
+        agent.transition_pause = _MINIWOB_TRANSITION_PAUSE
+    else:
+        agent.transition_pause = None
+
+    if _CHECKPOINT_DIR.value:
+        checkpoint_dir = _CHECKPOINT_DIR.value
+    else:
+        checkpoint_dir = checkpointer_lib.create_run_directory(_OUTPUT_PATH.value)
+
+    print(
+        f"Starting eval with agent {_AGENT_NAME.value} and writing to"
+        f" {checkpoint_dir}"
+    )
+    suite_utils.run(
+        suite,
+        agent,
+        checkpointer=checkpointer_lib.IncrementalCheckpointer(checkpoint_dir),
+        demo_mode=False,
+    )
+    print(
+        f"Finished running agent {_AGENT_NAME.value} on {_SUITE_FAMILY.value}"
+        f" family. Wrote to {checkpoint_dir}."
+    )
+    env.close()
+
+
+def main(argv: Sequence[str]) -> None:
+    del argv
+    _main()
+
+
+if __name__ == "__main__":
+    app.run(main)

baselines/Aria-UI/assets/aria_ui_framework_v4.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f5ee32f13abf04b44357801582680ebd8c371e80b4cebf5b8961d842c963e40
+size 2277714