ui_element_client.py

#!/usr/bin/env python3
"""
UI Element Detection API - Python Client Example
Demonstrates how to use the API to analyze screenshots
"""

import requests
import json
import base64
import argparse
from pathlib import Path
from PIL import Image
import io


class UIElementDetectionClient:
    def __init__(self, api_url="http://127.0.0.1:8001"):
        self.api_url = api_url
    
    def health_check(self):
        """Check if API is running."""
        try:
            # Try new API endpoint first
            response = requests.get(f"{self.api_url}/api/health", timeout=5)
            return response.json()
        except Exception:
            # Fallback to old endpoint
            try:
                response = requests.get(f"{self.api_url}/health", timeout=5)
                return response.json()
            except Exception as e:
                return {"error": str(e)}
    
    def analyze_image(self, image_path):
        """
        Analyze an image and get UI element coordinates.
        
        Args:
            image_path: Path to PNG image file
        
        Returns:
            Dictionary with analysis results
        """
        if not Path(image_path).exists():
            raise FileNotFoundError(f"Image not found: {image_path}")
        
        print(f"[Client] Analyzing: {image_path}")
        
        # Try new API endpoint first
        api_endpoint = f"{self.api_url}/api/analyze"
        print(f"[Client] Uploading to {api_endpoint}...")
        
        with open(image_path, 'rb') as f:
            files = {'file': f}
            try:
                response = requests.post(
                    api_endpoint,
                    files=files,
                    timeout=300
                )
            except Exception:
                # Fallback to old endpoint
                api_endpoint = f"{self.api_url}/analyze"
                print(f"[Client] Retrying with {api_endpoint}...")
                response = requests.post(
                    api_endpoint,
                    files=files,
                    timeout=300
                )
        
        if response.status_code != 200:
            raise Exception(f"API error: {response.status_code} - {response.text}")
        
        return response.json()
    
    def get_element_coordinates(self, image_path, save_outputs=False):
        """Analyze image and return clean coordinates."""
        result = self.analyze_image(image_path)
        
        if result['status'] != 'success':
            raise Exception(f"Analysis failed: {result}")
        
        print(f"\n[Results] Successfully analyzed image")
        print(f"  Total UI Elements: {result['analysis']['total_elements_detected']}")
        print(f"  Processing Time: {result['processing_time_seconds']:.2f}s")
        print(f"  Image Size: {result['image_info']['size']['width']}x{result['image_info']['size']['height']}")
        
        # Save outputs if requested
        if save_outputs:
            base_name = Path(image_path).stem
            
            # Save JSON
            json_file = f"{base_name}_coordinates.json"
            with open(json_file, 'w') as f:
                json.dump(result['analysis'], f, indent=2)
            print(f"  Saved JSON: {json_file}")
            
            # Save CSV
            csv_file = f"{base_name}_coordinates.csv"
            with open(csv_file, 'w') as f:
                f.write(result['exports']['csv_data'])
            print(f"  Saved CSV: {csv_file}")
            
            # Save visualization
            viz_file = f"{base_name}_visualization.png"
            viz_bytes = base64.b64decode(result['exports']['visualization_png_base64'])
            Image.open(io.BytesIO(viz_bytes)).save(viz_file)
            print(f"  Saved Visualization: {viz_file}")
        
        return result
    
    def get_element_by_id(self, image_path, element_id):
        """Get specific element coordinates by ID."""
        result = self.analyze_image(image_path)
        
        for elem in result['analysis']['elements']:
            if elem['template_id'] == element_id:
                return elem
        
        return None
    
    def find_elements_in_region(self, image_path, x1, y1, x2, y2):
        """Find all elements within a region."""
        result = self.analyze_image(image_path)
        
        elements = []
        for elem in result['analysis']['elements']:
            bbox = elem['bbox']
            # Check if element overlaps with region
            if (bbox['x1'] < x2 and bbox['x2'] > x1 and
                bbox['y1'] < y2 and bbox['y2'] > y1):
                elements.append(elem)
        
        return elements


def main():
    parser = argparse.ArgumentParser(description='UI Element Detection API Client')
    parser.add_argument('image', help='Path to image file')
    parser.add_argument('--api', default='http://127.0.0.1:8001', help='API URL')
    parser.add_argument('--save', action='store_true', help='Save output files')
    parser.add_argument('--element', help='Get specific element by ID')
    parser.add_argument('--region', nargs=4, type=int, metavar=('X1', 'Y1', 'X2', 'Y2'), 
                       help='Find elements in region')
    args = parser.parse_args()
    
    client = UIElementDetectionClient(args.api)
    
    # Check API health
    print("[Client] Checking API health...")
    health = client.health_check()
    if 'error' in health:
        print(f"[ERROR] API not available: {health['error']}")
        return
    print(f"[Client] API Status: {health['status']}")
    
    # Analyze image
    print()
    try:
        if args.element:
            # Get specific element
            result = client.analyze_image(args.image)
            element = None
            for elem in result['analysis']['elements']:
                if elem.get('element_id') == args.element or elem.get('template_id') == args.element:
                    element = elem
                    break
            
            if element:
                print(f"\n[Element: {args.element}]")
                print(f"  Position (center): ({element['center']['x']}, {element['center']['y']})")
                print(f"  Bounding Box: ({element['bbox']['x1']}, {element['bbox']['y1']}) -> ({element['bbox']['x2']}, {element['bbox']['y2']})")
                print(f"  Size: {element['bbox']['width']}x{element['bbox']['height']}")
                print(f"  Confidence: {element['confidence']:.4f}")
            else:
                print(f"[ERROR] Element '{args.element}' not found")
        
        elif args.region:
            # Find in region
            elements = client.find_elements_in_region(args.image, *args.region)
            print(f"\n[Found {len(elements)} elements in region {args.region}]")
            for elem in elements:
                elem_id = elem.get('element_id') or elem.get('template_id', 'unknown')
                x = elem.get('x') or elem.get('center', {}).get('x', 0)
                y = elem.get('y') or elem.get('center', {}).get('y', 0)
                print(f"  - {elem_id} @ ({x}, {y})")
        
        else:
            # Full analysis
            result = client.get_element_coordinates(args.image, save_outputs=args.save)
            
            print(f"\n[Top 5 Elements by Confidence]")
            for i, elem in enumerate(result['analysis']['elements'][:5], 1):
                elem_id = elem.get('element_id') or elem.get('template_id', f"elem_{i}")
                x = elem.get('x') or elem.get('center', {}).get('x', 0)
                y = elem.get('y') or elem.get('center', {}).get('y', 0)
                conf = elem.get('confidence', 0)
                print(f"  {i}. {elem_id} @ ({x}, {y}) - {conf:.4f}")
    
    except Exception as e:
        print(f"[ERROR] {str(e)}")
        import traceback
        traceback.print_exc()


if __name__ == "__main__":
    main()