baselines/Aria-UI/AndroidWorld/agents/aria_ui_utils.py

# Copyright 2024 The Aria-UI Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
from io import BytesIO
import requests
from PIL import Image
from openai import OpenAI
import numpy as np
import cv2
from android_world.agents.m3a_utils import _logical_to_physical

"""
Deploy Aria-UI with vLLM, then get the api_key and api_base from the deployment for directly API call.
"""
ariaui_api_key = "ariaui_api_key"
ariaui_api_base = "ariaui_api_base"

client = OpenAI(
    api_key=ariaui_api_key,
    base_url=ariaui_api_base,
)

models = client.models.list()
model = models.data[0].id

def encode_image_to_base64(image_path):
    pil_image = Image.open(image_path).convert('RGB')
    buffered = BytesIO()
    pil_image.save(buffered, format="JPEG")
    base64_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
    return base64_str

def encode_numpy_image_to_base64(image: np.ndarray) -> str:
    """Converts a numpy array image to base64 string.
    
    Args:
        image: Numpy array representing an image (height, width, channels)
        
    Returns:
        Base64 encoded string of the image
    """
    # Convert numpy array to bytes
    success, buffer = cv2.imencode('.jpg', image)
    if not success:
        raise ValueError("Failed to encode image to jpg format")
    
    # Convert bytes to base64 string
    image_bytes = buffer.tobytes()
    base64_string = base64.b64encode(image_bytes).decode('utf-8')
    
    return base64_string

def request_aria_ui(image: np.ndarray, prompt: str) -> str:
    image_base64 = encode_numpy_image_to_base64(image)
    chat_completion_from_url = client.chat.completions.create(
        messages=[{
            "role":
            "user",
            "content": [
                {
                    "type": "text",
                    "text": prompt
                },
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:image/jpeg;base64,{image_base64}"
                    },
                },
            ],
        }],
        model=model,
        max_tokens=512,
        stop=["<|im_end|>"],
        extra_body= {
            "split_image": True,
            "image_max_size": 980
        }
    )

    result = chat_completion_from_url.choices[0].message.content
    print(f"Chat completion output:{result}")
    return result


def add_ui_element_mark_coords(
    screenshot: np.ndarray,
    coords: tuple[int, int],  # Normalized coordinates in [0, 1000]
    logical_screen_size: tuple[int, int],
    physical_frame_boundary: tuple[int, int, int, int],
    orientation: int,
):
    """Add a red circle marker at the specified normalized coordinates.

    Args:
        screenshot: The screenshot as a numpy ndarray.
        coords: Normalized coordinates (x, y) in range [0, 1000].
        logical_screen_size: The logical screen size.
        physical_frame_boundary: The physical coordinates in portrait orientation
          for the upper left and lower right corner for the frame.
        orientation: The current screen orientation.
    """
    # Convert normalized coordinates to logical coordinates
    logical_point = (
        coords[0] * logical_screen_size[0] // 1000,
        coords[1] * logical_screen_size[1] // 1000
    )
    
    # Convert to physical coordinates
    physical_point = _logical_to_physical(
        logical_point,
        logical_screen_size,
        physical_frame_boundary,
        orientation,
    )

    # Draw a large red circle
    radius = 30  # Adjust size as needed
    cv2.circle(
        screenshot,
        physical_point,
        radius,
        color=(0, 0, 255),  # BGR format - Red
        thickness=3
    )

def convert_coords_to_physical(coords: tuple[int, int], logical_screen_size: tuple[int, int], physical_frame_boundary: tuple[int, int, int, int], orientation: int) -> tuple[int, int]:
    logical_point = (
        coords[0] * logical_screen_size[0] // 1000,
        coords[1] * logical_screen_size[1] // 1000
    )
    physical_point = _logical_to_physical(
        logical_point,
        logical_screen_size,
        physical_frame_boundary,
        orientation,
    )
    return physical_point