Update app.py

app.py

@@ -971,245 +971,280 @@ async def refresh_page(request: dict, token: str = Depends(verify_token)):
 # ===================== CLICK ELEMENT =====================
 @app.post("/click-element")
 async def click_element(request: dict, token: str = Depends(verify_token)):
-    """Click at specific coordinates on the page"""
+    """Click at specific coordinates using ActionChains for precision"""
     try:
         session_id = request.get("session_id")
         x = request.get("x", 0)
         y = request.get("y", 0)
-        
+
         if not session_id:
-            raise HTTPException(status_code=400, detail="session_id é obrigatório")
-        
+            raise HTTPException(status_code=400, detail="session_id required")
+
         session = session_manager.get_session(session_id)
+        if not session:
+            raise HTTPException(status_code=410, detail="Session expired")
+
         driver = session["driver"]
-        
-        # Get viewport size to scale coordinates
-        viewport = driver.execute_script("""
-            return {
-                width: window.innerWidth,
-                height: window.innerHeight,
-                scrollX: window.pageXOffset || document.documentElement.scrollLeft,
-                scrollY: window.pageYOffset || document.documentElement.scrollTop
-            };
-        """)
-        
-        # Click using JavaScript at exact coordinates
-        click_result = driver.execute_script("""
+
+        # Use ActionChains for precise clicking
+        from selenium.webdriver.common.action_chains import ActionChains
+
+        # Move to absolute position from top-left of viewport
+        # First scroll to top to reset position
+        viewport_width = driver.execute_script("return window.innerWidth;")
+        viewport_height = driver.execute_script("return window.innerHeight;")
+        scroll_top = driver.execute_script("return window.pageYOffset || document.documentElement.scrollTop;")
+        scroll_left = driver.execute_script("return window.pageXOffset || document.documentElement.scrollLeft;")
+
+        # The x,y from frontend are in 1920x1080 space
+        # Scale to actual viewport
+        scale_x = viewport_width / 1920.0
+        scale_y = viewport_height / 1080.0
+        actual_x = int(x * scale_x)
+        actual_y = int(y * scale_y)
+
+        logger.info(f"[CLICK] Input: ({x},{y}) Viewport: {viewport_width}x{viewport_height} Scale: ({scale_x:.2f},{scale_y:.2f}) Actual: ({actual_x},{actual_y}) Scroll: ({scroll_left},{scroll_top})")
+
+        # Get element at coordinates using JavaScript
+        clicked_info = driver.execute_script("""
             var x = arguments[0];
             var y = arguments[1];
-            var el = document.elementFromPoint(x, y);
-            var result = {
-                tagName: '',
-                text: '',
-                type: '',
-                name: '',
-                id: '',
-                href: '',
-                clicked: false
-            };
-            if (el) {
-                result.tagName = el.tagName.toLowerCase();
-                result.text = (el.textContent || '').trim().substring(0, 100);
-                result.type = el.type || '';
-                result.name = el.name || '';
-                result.id = el.id || '';
-                result.href = el.href || '';
-                
-                // Focus if it's an input
-                if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.tagName === 'SELECT') {
-                    el.focus();
-                    el.click();
-                } else {
-                    el.click();
+            var elem = document.elementFromPoint(x, y);
+            if (elem) {
+                // Click the element
+                elem.click();
+
+                // Also try focus if it's an input
+                if (elem.tagName === 'INPUT' || elem.tagName === 'TEXTAREA' || elem.tagName === 'SELECT') {
+                    elem.focus();
                 }
-                result.clicked = true;
+
+                return {
+                    tagName: elem.tagName,
+                    id: elem.id || '',
+                    name: elem.name || '',
+                    type: elem.type || '',
+                    text: (elem.textContent || '').substring(0, 100).trim(),
+                    href: elem.href || '',
+                    className: (elem.className || '').substring(0, 100),
+                    value: elem.value || ''
+                };
             }
-            return result;
-        """, int(x), int(y))
-        
-        time.sleep(0.5)
-        
-        # Check if page changed
-        new_url = driver.current_url
-        new_title = driver.title
-        
-        # Handle new tabs
-        if len(driver.window_handles) > 1:
-            driver.switch_to.window(driver.window_handles[-1])
-            new_url = driver.current_url
-            new_title = driver.title
-        
-        session["url"] = new_url
-        session["title"] = new_title
-        
-        screenshot = driver.get_screenshot_as_base64()
-        
-        return JSONResponse(content={
-            "status": "success",
-            "click": click_result,
-            "coordinates": {"x": x, "y": y},
-            "viewport": viewport,
-            "url": new_url,
-            "title": new_title,
+            return null;
+        """, actual_x, actual_y)
+
+        # If elementFromPoint didn't work, try ActionChains as fallback
+        if not clicked_info:
+            try:
+                actions = ActionChains(driver)
+                # Move to body first, then offset
+                body = driver.find_element("tag name", "body")
+                actions.move_to_element_with_offset(body, actual_x - viewport_width//2, actual_y - viewport_height//2).click().perform()
+                clicked_info = {"tagName": "unknown", "text": "ActionChains click"}
+            except Exception as ac_err:
+                logger.warning(f"[CLICK] ActionChains fallback failed: {ac_err}")
+                clicked_info = {"tagName": "unknown", "text": "Click at coordinates"}
+
+        import time
+        time.sleep(0.3)
+
+        # Check for new tabs
+        try:
+            if len(driver.window_handles) > 1:
+                driver.switch_to.window(driver.window_handles[-1])
+        except:
+            pass
+
+        current_url = driver.current_url
+        current_title = driver.title
+        session["url"] = current_url
+        session["title"] = current_title
+
+        import base64
+        screenshot = base64.b64encode(driver.get_screenshot_as_png()).decode('utf-8')
+
+        return {
+            "status": "clicked",
+            "clicked": clicked_info,
+            "coordinates": {"x": x, "y": y, "actual_x": actual_x, "actual_y": actual_y},
+            "viewport": {"width": viewport_width, "height": viewport_height},
+            "url": current_url,
+            "title": current_title,
             "screenshot": screenshot
-        })
-        
+        }
+
     except HTTPException:
         raise
     except Exception as e:
-        logger.error(f"[CLICK] Erro: {e}")
+        logger.error(f"[CLICK] Error: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
 # ===================== TYPE TEXT =====================
 @app.post("/type-text")
 async def type_text(request: dict, token: str = Depends(verify_token)):
-    """Type text into the currently focused element or find input and type"""
+    """Type text or press special keys using ActionChains"""
     try:
         session_id = request.get("session_id")
         text = request.get("text", "")
         press_enter = request.get("press_enter", False)
-        clear_first = request.get("clear_first", True)
-        selector = request.get("selector", "")
-        
+        clear_first = request.get("clear_first", False)
+        selector = request.get("selector", None)
+
         if not session_id:
-            raise HTTPException(status_code=400, detail="session_id é obrigatório")
-        if not text and not press_enter:
-            raise HTTPException(status_code=400, detail="text é obrigatório")
-        
+            raise HTTPException(status_code=400, detail="session_id required")
+
         session = session_manager.get_session(session_id)
+        if not session:
+            raise HTTPException(status_code=410, detail="Session expired")
+
         driver = session["driver"]
-        
+
+        from selenium.webdriver.common.action_chains import ActionChains
+        from selenium.webdriver.common.keys import Keys
+
+        # Map special Selenium unicode chars to actual Keys
+        special_keys = {
+            '\uE004': Keys.TAB,
+            '\uE00C': Keys.ESCAPE,
+            '\uE007': Keys.ENTER,
+            '\uE003': Keys.BACKSPACE,
+            '\uE006': Keys.RETURN,
+            '\uE012': Keys.ARROW_LEFT,
+            '\uE013': Keys.ARROW_UP,
+            '\uE014': Keys.ARROW_RIGHT,
+            '\uE015': Keys.ARROW_DOWN,
+            '\uE010': Keys.END,
+            '\uE011': Keys.HOME,
+            '\uE00D': Keys.SPACE,
+            '\uE017': Keys.DELETE,
+        }
+
         typed = False
-        element_info = {}
-        
-        # If selector provided, find and focus that element
-        if selector:
-            try:
-                el = driver.find_element(By.CSS_SELECTOR, selector)
-                el.click()
+        element_info = {"tagName": "none", "method": "none"}
+
+        # Check if text is a special key
+        is_special = text in special_keys
+
+        if is_special:
+            # Use ActionChains to press the actual key
+            actions = ActionChains(driver)
+            actions.send_keys(special_keys[text]).perform()
+            typed = True
+            key_name = {
+                '\uE004': 'Tab', '\uE00C': 'Escape', '\uE007': 'Enter',
+                '\uE003': 'Backspace', '\uE006': 'Return',
+                '\uE012': 'ArrowLeft', '\uE013': 'ArrowUp',
+                '\uE014': 'ArrowRight', '\uE015': 'ArrowDown',
+                '\uE010': 'End', '\uE011': 'Home',
+                '\uE00D': 'Space', '\uE017': 'Delete'
+            }.get(text, 'Special')
+            element_info = {"tagName": "body", "method": "ActionChains", "key": key_name}
+            logger.info(f"[TYPE] Special key: {key_name}")
+        else:
+            # Find element to type into
+            target = None
+
+            # Try selector first
+            if selector:
+                try:
+                    target = driver.find_element("css selector", selector)
+                except:
+                    pass
+
+            # Try active element
+            if not target:
+                try:
+                    target = driver.switch_to.active_element
+                    tag = target.tag_name.lower() if target.tag_name else ""
+                    if tag not in ["input", "textarea", "select", "div", "span"]:
+                        # Active element is not typeable, check if it has contenteditable
+                        editable = target.get_attribute("contenteditable")
+                        if editable != "true":
+                            target = None
+                except:
+                    target = None
+
+            # Fallback: find first visible input
+            if not target:
+                try:
+                    inputs = driver.find_elements("css selector", "input:not([type='hidden']):not([type='submit']):not([type='button']):not([type='checkbox']):not([type='radio']), textarea")
+                    for inp in inputs:
+                        if inp.is_displayed() and inp.is_enabled():
+                            target = inp
+                            break
+                except:
+                    pass
+
+            if target:
+                try:
+                    target.click()
+                except:
+                    pass
+
                 if clear_first:
-                    el.clear()
+                    try:
+                        target.clear()
+                    except:
+                        try:
+                            target.send_keys(Keys.CONTROL + "a")
+                            target.send_keys(Keys.DELETE)
+                        except:
+                            pass
+
                 if text:
-                    el.send_keys(text)
+                    target.send_keys(text)
+
                 if press_enter:
-                    el.send_keys(Keys.ENTER)
+                    target.send_keys(Keys.ENTER)
+
                 typed = True
                 element_info = {
-                    "tagName": el.tag_name,
-                    "type": el.get_attribute("type") or "",
-                    "name": el.get_attribute("name") or "",
-                    "id": el.get_attribute("id") or "",
-                    "method": "selector"
+                    "tagName": target.tag_name or "unknown",
+                    "type": target.get_attribute("type") or "",
+                    "name": target.get_attribute("name") or "",
+                    "id": target.get_attribute("id") or "",
+                    "method": "direct"
                 }
-            except Exception as se:
-                logger.warning(f"Selector '{selector}' failed: {se}")
-        
-        # Try typing into active/focused element
-        if not typed:
-            try:
-                result = driver.execute_script("""
-                    var el = document.activeElement;
-                    if (el && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.isContentEditable)) {
-                        return {
-                            tagName: el.tagName.toLowerCase(),
-                            type: el.type || '',
-                            name: el.name || '',
-                            id: el.id || '',
-                            isInput: true
-                        };
-                    }
-                    return { isInput: false };
-                """)
-                
-                if result.get("isInput"):
-                    active = driver.switch_to.active_element
-                    if clear_first:
-                        active.clear()
-                    if text:
-                        active.send_keys(text)
+                logger.info(f"[TYPE] Typed into <{element_info['tagName']}> name={element_info['name']} id={element_info['id']}")
+            else:
+                # Last resort: ActionChains send_keys to whatever is focused
+                if text:
+                    actions = ActionChains(driver)
+                    actions.send_keys(text)
                     if press_enter:
-                        active.send_keys(Keys.ENTER)
+                        actions.send_keys(Keys.ENTER)
+                    actions.perform()
                     typed = True
-                    element_info = result
-                    element_info["method"] = "active_element"
-            except Exception as ae:
-                logger.warning(f"Active element typing failed: {ae}")
-        
-        # Fallback: find first visible input and type
-        if not typed:
-            try:
-                inputs = driver.find_elements(By.CSS_SELECTOR, 
-                    "input[type='text']:not([style*='display: none']), "
-                    "input[type='email']:not([style*='display: none']), "
-                    "input[type='password']:not([style*='display: none']), "
-                    "input:not([type]):not([style*='display: none']), "
-                    "textarea:not([style*='display: none'])"
-                )
-                for inp in inputs:
-                    try:
-                        if inp.is_displayed() and inp.is_enabled():
-                            inp.click()
-                            if clear_first:
-                                inp.clear()
-                            if text:
-                                inp.send_keys(text)
-                            if press_enter:
-                                inp.send_keys(Keys.ENTER)
-                            typed = True
-                            element_info = {
-                                "tagName": inp.tag_name,
-                                "type": inp.get_attribute("type") or "",
-                                "name": inp.get_attribute("name") or "",
-                                "id": inp.get_attribute("id") or "",
-                                "method": "fallback_first_input"
-                            }
-                            break
-                    except:
-                        continue
-            except Exception as fe:
-                logger.warning(f"Fallback typing failed: {fe}")
-        
-        # Only press Enter if requested and nothing else was done
-        if not typed and press_enter:
-            try:
-                ActionChains(driver).send_keys(Keys.ENTER).perform()
-                typed = True
-                element_info = {"method": "action_chains_enter"}
-            except:
-                pass
-        
-        time.sleep(0.5)
-        
-        # Wait if enter was pressed (page might change)
-        if press_enter:
-            time.sleep(1.5)
-            try:
-                WebDriverWait(driver, 5).until(
-                    lambda d: d.execute_script("return document.readyState") == "complete"
-                )
-            except:
-                pass
-        
-        session["url"] = driver.current_url
-        session["title"] = driver.title
-        
-        screenshot = driver.get_screenshot_as_base64()
-        
-        return JSONResponse(content={
-            "status": "success",
+                    element_info = {"tagName": "active", "method": "ActionChains"}
+                    logger.info("[TYPE] Used ActionChains fallback")
+
+        import time
+        time.sleep(0.3)
+
+        current_url = driver.current_url
+        current_title = driver.title
+        session["url"] = current_url
+        session["title"] = current_title
+
+        import base64
+        screenshot = base64.b64encode(driver.get_screenshot_as_png()).decode('utf-8')
+
+        return {
+            "status": "typed" if typed else "no_target",
             "typed": typed,
-            "text_sent": text,
-            "enter_pressed": press_enter,
+            "text_sent": text if not is_special else "",
+            "press_enter": press_enter,
+            "special_key": is_special,
             "element": element_info,
-            "url": driver.current_url,
-            "title": driver.title,
+            "url": current_url,
+            "title": current_title,
             "screenshot": screenshot
-        })
-        
+        }
+
     except HTTPException:
         raise
     except Exception as e:
-        logger.error(f"[TYPE] Erro: {e}")
+        logger.error(f"[TYPE] Error: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
 # ============================================================