Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,24 +1,23 @@
|
|
| 1 |
"""
|
| 2 |
-
THE Z AI — Computer Mode Server
|
| 3 |
=====================================================================================
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
-
|
| 14 |
-
|
| 15 |
-
-
|
| 16 |
-
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
- v12: الإحداثيات مأخوذة مباشرة من DOM (دقة 100%)، LLM يختار العنصر بـ index
|
| 22 |
|
| 23 |
📋 كيف يشتغل CDP هنا:
|
| 24 |
- Chromium يُشغَّل مع --remote-debugging-port=XXXX (منفذ خاص لكل مستخدم)
|
|
@@ -26,7 +25,7 @@ v12 يضيف فوق v11 طبقة رؤية ذكية مستوحاة من Manus:
|
|
| 26 |
- نقرا DOM.getDocument + DOM.querySelectorAll لكل العناصر القابلة للنقر
|
| 27 |
- نرجع قائمة JSON منظمة: [{id, tag, text, x, y, w, h, href, type}, ...]
|
| 28 |
|
| 29 |
-
🔄 التوافق الخلفي: كل أدوات v11 (mouse_click, keyboard_type, إلخ) تعمل بدون تغيير
|
| 30 |
"""
|
| 31 |
|
| 32 |
import asyncio
|
|
@@ -457,98 +456,40 @@ async def cdp_click_element(display: str, element_id: int, elements: list) -> di
|
|
| 457 |
x, y = new_el["x"], new_el["y"]
|
| 458 |
|
| 459 |
# النقر بالإحداثيات الحقيقية عبر xdotool (نفس محرك v11)
|
| 460 |
-
return {"ok": True, "x": x, "y": y, "
|
| 461 |
|
| 462 |
|
| 463 |
-
async def
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
for el in elements:
|
| 473 |
-
hay = " ".join([
|
| 474 |
-
str(el.get("text", "")),
|
| 475 |
-
str(el.get("tag", "")),
|
| 476 |
-
str(el.get("role", "")),
|
| 477 |
-
str(el.get("href", "")),
|
| 478 |
-
str(el.get("type", "")),
|
| 479 |
-
str(el.get("label", "")),
|
| 480 |
-
]).lower()
|
| 481 |
-
if q not in hay:
|
| 482 |
-
continue
|
| 483 |
-
score = 0
|
| 484 |
-
text = str(el.get("text", "")).lower()
|
| 485 |
-
role = str(el.get("role", "")).lower()
|
| 486 |
-
tag = str(el.get("tag", "")).lower()
|
| 487 |
-
href = str(el.get("href", "")).lower()
|
| 488 |
-
if q == text: score += 50
|
| 489 |
-
if q in text: score += 30
|
| 490 |
-
if q in role: score += 12
|
| 491 |
-
if q in tag: score += 8
|
| 492 |
-
if q in href: score += 10
|
| 493 |
-
if el.get("in_viewport", True): score += 6
|
| 494 |
-
# shorter labels are often better click targets
|
| 495 |
-
score += max(0, 12 - min(len(text), 12))
|
| 496 |
-
scored.append((score, el))
|
| 497 |
-
|
| 498 |
-
scored.sort(key=lambda x: (-x[0], x[1].get("id", 10**9)))
|
| 499 |
-
matches = [el for _, el in scored[:max(1, min(int(limit or 8), 25))]]
|
| 500 |
-
return {
|
| 501 |
-
"matches": matches,
|
| 502 |
-
"count": len(matches),
|
| 503 |
-
"query": query,
|
| 504 |
-
**{k: state.get(k) for k in ("page_title", "page_url", "viewport_w", "viewport_h", "scroll_y", "total_height")},
|
| 505 |
-
}
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
async def cdp_type_into_element(display: str, element_id: int, text: str, elements: list) -> dict:
|
| 509 |
-
"""يُفعّل عنصر إدخال ثم يكتب فيه. يعتمد على آخر DOM snapshot."""
|
| 510 |
el = next((e for e in elements if e.get("id") == element_id), None)
|
| 511 |
if not el:
|
| 512 |
return {"error": f"Element {element_id} not found in elements list"}
|
| 513 |
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 517 |
|
| 518 |
-
|
| 519 |
if not el.get("in_viewport", True):
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
await sess.send("Runtime.evaluate", {
|
| 523 |
-
"expression": scroll_expr,
|
| 524 |
-
"returnByValue": True,
|
| 525 |
-
"awaitPromise": True,
|
| 526 |
-
"timeout": 5000,
|
| 527 |
-
}, timeout=6.0)
|
| 528 |
-
await asyncio.sleep(0.25)
|
| 529 |
-
refreshed = await cdp_get_elements(display)
|
| 530 |
-
new_el = next((e for e in refreshed.get("elements", []) if e.get("id") == element_id), None)
|
| 531 |
if new_el:
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
x, y = int(el.get("x", 0)), int(el.get("y", 0))
|
| 535 |
-
tag = str(el.get("tag", "")).lower()
|
| 536 |
-
el_type = str(el.get("type", "")).lower()
|
| 537 |
-
is_text_input = tag in {"input", "textarea"} or el.get("role") == "textbox" or el.get("contenteditable")
|
| 538 |
|
| 539 |
-
|
| 540 |
-
await asyncio.sleep(0.05)
|
| 541 |
-
return {
|
| 542 |
-
"ok": True,
|
| 543 |
-
"x": x,
|
| 544 |
-
"y": y,
|
| 545 |
-
"page_x": el.get("page_x", x),
|
| 546 |
-
"page_y": el.get("page_y", y),
|
| 547 |
-
"tag": tag,
|
| 548 |
-
"type": el_type,
|
| 549 |
-
"is_text_input": bool(is_text_input),
|
| 550 |
-
"text": str(text),
|
| 551 |
-
}
|
| 552 |
|
| 553 |
|
| 554 |
async def cdp_get_page_text(display: str) -> dict:
|
|
@@ -601,6 +542,48 @@ async def cdp_execute_js(display: str, js_code: str) -> dict:
|
|
| 601 |
return {"result": r.get("value"), "type": r.get("type", "undefined")}
|
| 602 |
|
| 603 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 604 |
async def cdp_navigate(display: str, url: str) -> dict:
|
| 605 |
"""يُنقل المتصفح لرابط معين عبر CDP — أسرع وأكثر موثوقية من xdotool."""
|
| 606 |
sess = await _get_cdp_session(display)
|
|
@@ -964,8 +947,6 @@ async def get_or_create_user_session(user_id: str, ws: WebSocket) -> dict:
|
|
| 964 |
"created": time.time(),
|
| 965 |
"consecutive_capture_failures": 0, # عدّاد فشل الالتقاط المتتالي — يُصفَّر عند أي نجاح أو reset
|
| 966 |
"cdp_port": _get_cdp_port(display), # v12: منفذ CDP الخاص بهذا المستخدم
|
| 967 |
-
"last_page_url": "", # v12: آخر URL تم جمع العناصر منه
|
| 968 |
-
"last_page_title": "",
|
| 969 |
"last_elements": [], # v12: آخر قائمة عناصر DOM (للـ click_element)
|
| 970 |
}
|
| 971 |
_user_displays[user_id] = sess
|
|
@@ -1560,7 +1541,7 @@ async def run_cmd_smart(cmd: str, display: str, timeout: int = 60) -> dict:
|
|
| 1560 |
# ── FastAPI App ───────────────────────────────────────────────
|
| 1561 |
# ════════════════════════════════════════════════════════════════
|
| 1562 |
|
| 1563 |
-
app = FastAPI(title="Z-Computer-Mode
|
| 1564 |
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True,
|
| 1565 |
allow_methods=["*"], allow_headers=["*"])
|
| 1566 |
|
|
@@ -1631,8 +1612,8 @@ def _format_elements_for_llm(elements: list) -> str:
|
|
| 1631 |
lines.append(_format_single_element(el))
|
| 1632 |
|
| 1633 |
lines.append(f"\n💡 To click element N: use action='browser_click_element' with id=N")
|
| 1634 |
-
lines.append("💡 To type into an input: use action='browser_type_into_element' with id=N")
|
| 1635 |
-
lines.append("💡 To
|
| 1636 |
return "\n".join(lines)
|
| 1637 |
|
| 1638 |
|
|
@@ -2234,12 +2215,12 @@ async def handle_action(ws: WebSocket, msg: dict, sess: dict):
|
|
| 2234 |
result = await cdp_get_elements(display)
|
| 2235 |
elements = result.get("elements", [])
|
| 2236 |
|
| 2237 |
-
# خزّن قائمة العناصر
|
|
|
|
| 2238 |
with _display_lock:
|
| 2239 |
if sess["user_id"] in _user_displays:
|
| 2240 |
_user_displays[sess["user_id"]]["last_elements"] = elements
|
| 2241 |
-
_user_displays[sess["user_id"]]["
|
| 2242 |
-
_user_displays[sess["user_id"]]["last_page_title"] = result.get("page_title", "")
|
| 2243 |
|
| 2244 |
await send({
|
| 2245 |
"type": "browser_elements",
|
|
@@ -2263,9 +2244,9 @@ async def handle_action(ws: WebSocket, msg: dict, sess: dict):
|
|
| 2263 |
element_id = int(data.get("id", -1))
|
| 2264 |
|
| 2265 |
with _display_lock:
|
| 2266 |
-
|
| 2267 |
-
elements =
|
| 2268 |
-
|
| 2269 |
|
| 2270 |
if element_id < 0 or not elements:
|
| 2271 |
await send({
|
|
@@ -2274,10 +2255,12 @@ async def handle_action(ws: WebSocket, msg: dict, sess: dict):
|
|
| 2274 |
})
|
| 2275 |
return
|
| 2276 |
|
| 2277 |
-
|
| 2278 |
-
|
| 2279 |
-
|
| 2280 |
-
|
|
|
|
|
|
|
| 2281 |
return
|
| 2282 |
|
| 2283 |
click_info = await cdp_click_element(display, element_id, elements)
|
|
@@ -2302,403 +2285,53 @@ async def handle_action(ws: WebSocket, msg: dict, sess: dict):
|
|
| 2302 |
asyncio.create_task(shot_bg("after element click", delay=0.8, extra_shot_delay=2.0))
|
| 2303 |
|
| 2304 |
# ── browser_type_into_element ────────────────────────────
|
| 2305 |
-
# كت
|
|
|
|
| 2306 |
elif action == "browser_type_into_element":
|
| 2307 |
element_id = int(data.get("id", -1))
|
| 2308 |
-
|
|
|
|
|
|
|
| 2309 |
with _display_lock:
|
| 2310 |
-
|
| 2311 |
-
elements =
|
| 2312 |
-
|
|
|
|
| 2313 |
if element_id < 0 or not elements:
|
| 2314 |
-
await send({
|
|
|
|
|
|
|
|
|
|
| 2315 |
return
|
| 2316 |
-
|
| 2317 |
-
|
| 2318 |
-
|
| 2319 |
-
|
|
|
|
|
|
|
|
|
|
| 2320 |
return
|
| 2321 |
-
|
|
|
|
| 2322 |
if "error" in type_info:
|
| 2323 |
await send({"type": "error", "msg": type_info["error"]})
|
| 2324 |
return
|
| 2325 |
-
|
| 2326 |
-
|
|
|
|
|
|
|
| 2327 |
await xdo(["click", "1"], display)
|
| 2328 |
-
await asyncio.sleep(0.
|
| 2329 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2330 |
await send({
|
| 2331 |
"type": "ack",
|
| 2332 |
"action": "browser_type_into_element",
|
| 2333 |
"element_id": element_id,
|
| 2334 |
-
"
|
| 2335 |
-
"
|
| 2336 |
-
"text_len": len(text_value),
|
| 2337 |
-
})
|
| 2338 |
-
asyncio.create_task(shot_bg("after element type", delay=0.8, extra_shot_delay=2.0))
|
| 2339 |
-
|
| 2340 |
-
# ── browser_find_element ─────────────────────────────────
|
| 2341 |
-
elif action == "browser_find_element":
|
| 2342 |
-
query = str(data.get("query", "")).strip()
|
| 2343 |
-
limit = int(data.get("limit", 8) or 8)
|
| 2344 |
-
if not query:
|
| 2345 |
-
await send({"type": "error", "msg": "No query provided"})
|
| 2346 |
-
return
|
| 2347 |
-
found = await cdp_find_elements(display, query, limit=limit)
|
| 2348 |
-
matches = found.get("matches", [])
|
| 2349 |
-
with _display_lock:
|
| 2350 |
-
if sess["user_id"] in _user_displays:
|
| 2351 |
-
_user_displays[sess["user_id"]]["last_elements"] = found.get("matches", [])
|
| 2352 |
-
_user_displays[sess["user_id"]]["last_page_url"] = found.get("page_url", "")
|
| 2353 |
-
_user_displays[sess["user_id"]]["last_page_title"] = found.get("page_title", "")
|
| 2354 |
-
await send({
|
| 2355 |
-
"type": "browser_find_result",
|
| 2356 |
-
"query": query,
|
| 2357 |
-
"count": len(matches),
|
| 2358 |
-
"matches": matches,
|
| 2359 |
-
"summary": _format_elements_for_llm(matches),
|
| 2360 |
-
"page_title": found.get("page_title", ""),
|
| 2361 |
-
"page_url": found.get("page_url", ""),
|
| 2362 |
-
"error": found.get("error"),
|
| 2363 |
-
})
|
| 2364 |
-
|
| 2365 |
-
# ── browser_get_page_text ────────────────────────────────
|
| 2366 |
-
# نص الصفحة الكامل — مثل Manus shell_view لكن للمتصفح
|
| 2367 |
-
elif action == "browser_get_page_text":
|
| 2368 |
-
result = await cdp_get_page_text(display)
|
| 2369 |
-
await send({
|
| 2370 |
-
"type": "browser_page_text",
|
| 2371 |
-
"text": result.get("text", ""),
|
| 2372 |
-
"title": result.get("title", ""),
|
| 2373 |
-
"url": result.get("url", ""),
|
| 2374 |
-
"char_count": result.get("char_count", 0),
|
| 2375 |
-
"error": result.get("error"),
|
| 2376 |
-
})
|
| 2377 |
-
|
| 2378 |
-
# ── browser_get_state ─────────────────────────────────────
|
| 2379 |
-
# لقطة واحدة تجمع screenshot + DOM state + النص + العناصر
|
| 2380 |
-
elif action == "browser_get_state":
|
| 2381 |
-
scale = float(data.get("scale", 0.65) or 0.65)
|
| 2382 |
-
quality = int(data.get("quality", 72) or 72)
|
| 2383 |
-
result = await asyncio.to_thread(capture_with_grid, display, scale, quality)
|
| 2384 |
-
dom = await cdp_get_elements(display)
|
| 2385 |
-
txt = await cdp_get_page_text(display)
|
| 2386 |
-
elements = dom.get("elements", [])
|
| 2387 |
-
with _display_lock:
|
| 2388 |
-
if sess["user_id"] in _user_displays:
|
| 2389 |
-
_user_displays[sess["user_id"]]["last_elements"] = elements
|
| 2390 |
-
_user_displays[sess["user_id"]]["last_page_url"] = dom.get("page_url", "")
|
| 2391 |
-
_user_displays[sess["user_id"]]["last_page_title"] = dom.get("page_title", "")
|
| 2392 |
-
await send({
|
| 2393 |
-
"type": "browser_state",
|
| 2394 |
-
"image": result["data"],
|
| 2395 |
-
"image_grid": result.get("data_grid", ""),
|
| 2396 |
-
"image_grid2": result.get("data_grid2", ""),
|
| 2397 |
-
"screen_width": result["width"],
|
| 2398 |
-
"screen_height": result["height"],
|
| 2399 |
-
"mouse_x": result["mouse_x"],
|
| 2400 |
-
"mouse_y": result["mouse_y"],
|
| 2401 |
-
"elements": elements,
|
| 2402 |
-
"count": len(elements),
|
| 2403 |
-
"summary": _format_elements_for_llm(elements),
|
| 2404 |
-
"page_title": dom.get("page_title", txt.get("title", "")),
|
| 2405 |
-
"page_url": dom.get("page_url", txt.get("url", "")),
|
| 2406 |
-
"page_text": txt.get("text", ""),
|
| 2407 |
-
"char_count": txt.get("char_count", 0),
|
| 2408 |
-
"error": dom.get("error") or txt.get("error"),
|
| 2409 |
-
})
|
| 2410 |
-
|
| 2411 |
-
# ── browser_execute_js ───────────────────────────────────
|
| 2412 |
-
# تنفيذ JavaScript — مثل Manus browser_console_exec
|
| 2413 |
-
elif action == "browser_execute_js":
|
| 2414 |
-
js_code = data.get("code", "")
|
| 2415 |
-
if not js_code:
|
| 2416 |
-
await send({"type": "error", "msg": "No JS code provided"})
|
| 2417 |
-
return
|
| 2418 |
-
result = await cdp_execute_js(display, js_code)
|
| 2419 |
-
await send({
|
| 2420 |
-
"type": "browser_js_result",
|
| 2421 |
-
"result": result.get("result"),
|
| 2422 |
-
"type_name": result.get("type", "undefined"),
|
| 2423 |
-
"error": result.get("error"),
|
| 2424 |
})
|
| 2425 |
-
|
| 2426 |
-
# ── browser_navigate ─────────────────────────────────────
|
| 2427 |
-
# انتقال سريع لرابط عبر CDP بدون إعادة فتح المتصفح
|
| 2428 |
-
elif action == "browser_navigate":
|
| 2429 |
-
url = _safe_search(data.get("url", ""))
|
| 2430 |
-
if not url:
|
| 2431 |
-
await send({"type": "error", "msg": "No URL provided"})
|
| 2432 |
-
return
|
| 2433 |
-
|
| 2434 |
-
if BROWSER_SUPPORTS_CDP:
|
| 2435 |
-
result = await cdp_navigate(display, url)
|
| 2436 |
-
if "ok" in result:
|
| 2437 |
-
await send({"type": "ack", "action": "browser_navigate", "url": url, "cdp": True})
|
| 2438 |
-
# انتظر تحميل الصفحة ثم التقط
|
| 2439 |
-
asyncio.create_task(shot_bg("after cdp navigate", delay=1.5, extra_shot_delay=2.5))
|
| 2440 |
-
else:
|
| 2441 |
-
# fallback لـ xdotool إذا فشل CDP
|
| 2442 |
-
await xdo(["key", "--clearmodifiers", "ctrl+l"], display)
|
| 2443 |
-
await asyncio.sleep(0.2)
|
| 2444 |
-
await type_smart(url, display)
|
| 2445 |
-
await xdo(["key", "--clearmodifiers", "Return"], display)
|
| 2446 |
-
await send({"type": "ack", "action": "browser_navigate", "url": url, "cdp": False})
|
| 2447 |
-
else:
|
| 2448 |
-
await xdo(["key", "--clearmodifiers", "ctrl+l"], display)
|
| 2449 |
-
await asyncio.sleep(0.2)
|
| 2450 |
-
await type_smart(url, display)
|
| 2451 |
-
await xdo(["key", "--clearmodifiers", "Return"], display)
|
| 2452 |
-
await send({"type": "ack", "action": "browser_navigate", "url": url, "cdp": False})
|
| 2453 |
-
# ── تعطيل الالتقاط التلقائي — shot_bg أعلاه يتكفل بها ──
|
| 2454 |
-
|
| 2455 |
-
# ── browser_scroll_to_element ────────────────────────────
|
| 2456 |
-
# تمرير الصفحة لعنصر معين بـ index
|
| 2457 |
-
elif action == "browser_scroll_to_element":
|
| 2458 |
-
element_id = int(data.get("id", -1))
|
| 2459 |
-
with _display_lock:
|
| 2460 |
-
state = _user_displays.get(sess["user_id"], {})
|
| 2461 |
-
elements = state.get("last_elements", [])
|
| 2462 |
-
last_url = state.get("last_page_url", "")
|
| 2463 |
-
el = next((e for e in elements if e.get("id") == element_id), None)
|
| 2464 |
-
if not el:
|
| 2465 |
-
await send({"type": "error", "msg": f"Element {element_id} not found"})
|
| 2466 |
-
return
|
| 2467 |
-
current = await cdp_get_elements(display)
|
| 2468 |
-
current_url = current.get("page_url", "")
|
| 2469 |
-
if last_url and current_url and last_url != current_url:
|
| 2470 |
-
await send({"type": "error", "msg": "Cached DOM is stale. Call browser_get_elements again on the current page."})
|
| 2471 |
-
return
|
| 2472 |
-
page_y = int(el.get("page_y") or el.get("y") or 0)
|
| 2473 |
-
await cdp_execute_js(display,
|
| 2474 |
-
f"window.scrollTo({{top: Math.max(0, {max(page_y - 350, 0)}), behavior:'instant'}}); true;"
|
| 2475 |
-
)
|
| 2476 |
-
await asyncio.sleep(0.4)
|
| 2477 |
-
await send({"type": "ack", "action": "browser_scroll_to_element", "element_id": element_id})
|
| 2478 |
-
asyncio.create_task(shot_bg("after scroll to element", delay=0.8))
|
| 2479 |
-
|
| 2480 |
-
# ── unknown ──────────────────────────────────────────────
|
| 2481 |
-
else:
|
| 2482 |
-
await send({"type": "error", "msg": f"Unknown action: '{action}'"})
|
| 2483 |
-
|
| 2484 |
-
|
| 2485 |
-
# ════════════════════════════════════════════════════════════════
|
| 2486 |
-
# ── WebSocket Endpoint ────────────────────────────────────────
|
| 2487 |
-
# ════════════════════════════════════════════════════════════════
|
| 2488 |
-
|
| 2489 |
-
@app.websocket("/ws")
|
| 2490 |
-
async def websocket_endpoint(
|
| 2491 |
-
ws: WebSocket,
|
| 2492 |
-
user_id: str = Query(default="anonymous")
|
| 2493 |
-
):
|
| 2494 |
-
await ws.accept()
|
| 2495 |
-
|
| 2496 |
-
# الحصول على session المستخدم أو إنشاء واحدة جديدة
|
| 2497 |
-
sess = await get_or_create_user_session(user_id, ws)
|
| 2498 |
-
|
| 2499 |
-
# تأكد أن Xvfb يعمل لهذا المستخدم
|
| 2500 |
-
await ensure_xvfb_for_session(sess)
|
| 2501 |
-
|
| 2502 |
-
display = sess["display"]
|
| 2503 |
-
|
| 2504 |
-
async def _heartbeat():
|
| 2505 |
-
while True:
|
| 2506 |
-
await asyncio.sleep(20)
|
| 2507 |
-
try:
|
| 2508 |
-
await ws.send_text(json.dumps({"type": "ping", "ts": int(time.time()*1000)}))
|
| 2509 |
-
except Exception:
|
| 2510 |
-
break
|
| 2511 |
-
|
| 2512 |
-
hb_task = asyncio.create_task(_heartbeat())
|
| 2513 |
-
|
| 2514 |
-
try:
|
| 2515 |
-
w, h = await asyncio.to_thread(_get_screen_size, display)
|
| 2516 |
-
await ws.send_text(json.dumps({
|
| 2517 |
-
"type": "connected",
|
| 2518 |
-
"screen_width": w, "screen_height": h,
|
| 2519 |
-
"browser": BROWSER,
|
| 2520 |
-
"display": display,
|
| 2521 |
-
"user_id": user_id,
|
| 2522 |
-
"session_id": id(ws),
|
| 2523 |
-
"msg": f"Z Computer Mode v11 | User: {user_id} | Display: {display} | Browser: {BROWSER} | Screen: {w}x{h}",
|
| 2524 |
-
}, ensure_ascii=False))
|
| 2525 |
-
|
| 2526 |
-
# لقطة شاشة أولية
|
| 2527 |
-
result = await asyncio.to_thread(capture_with_grid, display, 0.65, 72)
|
| 2528 |
-
if result["data"]:
|
| 2529 |
-
await ws.send_text(json.dumps({
|
| 2530 |
-
"type": "screenshot",
|
| 2531 |
-
"data": result["data"],
|
| 2532 |
-
"data_grid": result.get("data_grid", ""),
|
| 2533 |
-
"data_grid2": result.get("data_grid2", ""),
|
| 2534 |
-
"data_grid3": result.get("data_grid3", ""),
|
| 2535 |
-
"ts": int(time.time() * 1000),
|
| 2536 |
-
"label": "Initial screen",
|
| 2537 |
-
"screen_width": result["width"],
|
| 2538 |
-
"screen_height": result["height"],
|
| 2539 |
-
"mouse_x": result["mouse_x"],
|
| 2540 |
-
"mouse_y": result["mouse_y"],
|
| 2541 |
-
"has_grid": True,
|
| 2542 |
-
}, ensure_ascii=False))
|
| 2543 |
-
sess["last_bg_hash"] = _frame_hash(result["data"])
|
| 2544 |
-
|
| 2545 |
-
except Exception as e:
|
| 2546 |
-
print(f"[ws:{user_id}] init error: {e}")
|
| 2547 |
-
|
| 2548 |
-
try:
|
| 2549 |
-
while True:
|
| 2550 |
-
raw = await ws.receive_text()
|
| 2551 |
-
try:
|
| 2552 |
-
msg = json.loads(raw)
|
| 2553 |
-
if msg.get("type") == "pong":
|
| 2554 |
-
continue
|
| 2555 |
-
await handle_action(ws, msg, sess)
|
| 2556 |
-
except json.JSONDecodeError:
|
| 2557 |
-
pass
|
| 2558 |
-
except WebSocketDisconnect:
|
| 2559 |
-
pass
|
| 2560 |
-
except Exception as e:
|
| 2561 |
-
print(f"[ws:{user_id}] error: {e}")
|
| 2562 |
-
finally:
|
| 2563 |
-
hb_task.cancel()
|
| 2564 |
-
await destroy_user_ws(user_id, ws)
|
| 2565 |
-
|
| 2566 |
-
|
| 2567 |
-
# ════════════════════════════════════════════════════════════════
|
| 2568 |
-
# ── REST Endpoints ────────────────────────────────────────────
|
| 2569 |
-
# ════════════════════════════════════════════════════════════════
|
| 2570 |
-
|
| 2571 |
-
@app.get("/screenshot")
|
| 2572 |
-
async def rest_screenshot(user_id: str = "anonymous"):
|
| 2573 |
-
with _display_lock:
|
| 2574 |
-
sess = _user_displays.get(user_id)
|
| 2575 |
-
display = sess["display"] if sess else f":{DISPLAY_BASE}"
|
| 2576 |
-
result = await asyncio.to_thread(capture_with_grid, display, 0.7, 75)
|
| 2577 |
-
return JSONResponse({
|
| 2578 |
-
"image": result["data"],
|
| 2579 |
-
"image_grid": result.get("data_grid", ""),
|
| 2580 |
-
"image_grid2": result.get("data_grid2", ""),
|
| 2581 |
-
"image_grid3": result.get("data_grid3", ""),
|
| 2582 |
-
"ts": int(time.time() * 1000),
|
| 2583 |
-
"screen_width": result["width"],
|
| 2584 |
-
"screen_height": result["height"],
|
| 2585 |
-
"mouse_x": result["mouse_x"],
|
| 2586 |
-
"mouse_y": result["mouse_y"],
|
| 2587 |
-
"has_grid": True,
|
| 2588 |
-
"display": display,
|
| 2589 |
-
"user_id": user_id,
|
| 2590 |
-
})
|
| 2591 |
-
|
| 2592 |
-
@app.post("/terminal")
|
| 2593 |
-
async def rest_terminal(body: dict):
|
| 2594 |
-
user_id = body.get("user_id", "anonymous")
|
| 2595 |
-
with _display_lock:
|
| 2596 |
-
sess = _user_displays.get(user_id)
|
| 2597 |
-
display = sess["display"] if sess else f":{DISPLAY_BASE}"
|
| 2598 |
-
return JSONResponse(await run_cmd_smart(body.get("cmd", ""), display, body.get("timeout", 60)))
|
| 2599 |
-
|
| 2600 |
-
@app.get("/health")
|
| 2601 |
-
async def health():
|
| 2602 |
-
with _display_lock:
|
| 2603 |
-
n = len(_user_displays)
|
| 2604 |
-
users = [
|
| 2605 |
-
{
|
| 2606 |
-
"user_id": uid,
|
| 2607 |
-
"display": s["display"],
|
| 2608 |
-
"cdp_port": s.get("cdp_port"),
|
| 2609 |
-
"connected": bool(s.get("active_ws")),
|
| 2610 |
-
"elements_cached": len(s.get("last_elements", [])),
|
| 2611 |
-
}
|
| 2612 |
-
for uid, s in _user_displays.items()
|
| 2613 |
-
]
|
| 2614 |
-
with _cdp_sessions_lock:
|
| 2615 |
-
cdp_active = sum(1 for s in _cdp_sessions.values() if s._ws)
|
| 2616 |
-
return {
|
| 2617 |
-
"status": "ok",
|
| 2618 |
-
"version": "v12-cdp-dom-elements",
|
| 2619 |
-
"browser": BROWSER,
|
| 2620 |
-
"cdp_supported": BROWSER_SUPPORTS_CDP,
|
| 2621 |
-
"cdp_library": _CDP_AVAILABLE,
|
| 2622 |
-
"active_users": n,
|
| 2623 |
-
"cdp_sessions_active": cdp_active,
|
| 2624 |
-
"users": users,
|
| 2625 |
-
}
|
| 2626 |
-
|
| 2627 |
-
|
| 2628 |
-
# ════════════════════════════════════════════════════════════════
|
| 2629 |
-
# ── Background Tasks ──────────────────────────────────────────
|
| 2630 |
-
# ════════════════════════════════════════════════════════════════
|
| 2631 |
-
|
| 2632 |
-
async def _cleanup_tmp():
|
| 2633 |
-
"""ينظّف /tmp كل 5 دقائق. ملاحظة v13: محرك الالتقاط
|
| 2634 |
-
الجديد (import-pipe/xlib-direct/xwd) لا يكتب أي ملفات zss_* على الإطلاق (كلشيء
|
| 2635 |
-
في الذاكرة عبر pipe)، فهذه الدالة أصبحت غير ضرورية فعلياً للقطات
|
| 2636 |
-
الشاشة، لكنها تُرك كشبكة أمان إضافية (مثلاً لو أضيف كود مستقبلاً
|
| 2637 |
-
يكتب ملفات مؤقتة بنفس البادئة)."""
|
| 2638 |
-
while True:
|
| 2639 |
-
await asyncio.sleep(300)
|
| 2640 |
-
try:
|
| 2641 |
-
subprocess.run(
|
| 2642 |
-
["find", "/tmp", "-name", "zss_*", "-mmin", "+10", "-delete"],
|
| 2643 |
-
capture_output=True, timeout=10
|
| 2644 |
-
)
|
| 2645 |
-
except Exception as e:
|
| 2646 |
-
print(f"[cleanup] {e}")
|
| 2647 |
-
|
| 2648 |
-
|
| 2649 |
-
async def _cleanup_idle_sessions():
|
| 2650 |
-
"""
|
| 2651 |
-
يُزيل sessions المستخدمين غير النشطين (لا اتصال منذ أكثر من ساعة)
|
| 2652 |
-
لتحرير الـ displays والذاكرة.
|
| 2653 |
-
"""
|
| 2654 |
-
while True:
|
| 2655 |
-
await asyncio.sleep(1800) # كل 30 دقيقة
|
| 2656 |
-
now = time.time()
|
| 2657 |
-
to_remove = []
|
| 2658 |
-
with _display_lock:
|
| 2659 |
-
for uid, sess in list(_user_displays.items()):
|
| 2660 |
-
if sess.get("active_ws"):
|
| 2661 |
-
continue # لا تحذف sessions النشطة
|
| 2662 |
-
if now - sess.get("created", now) > 3600: # أكثر من ساعة
|
| 2663 |
-
to_remove.append((uid, sess))
|
| 2664 |
-
|
| 2665 |
-
for uid, sess in to_remove:
|
| 2666 |
-
print(f"[cleanup] 🗑️ Removing idle session for '{uid}' on {sess['display']}")
|
| 2667 |
-
_kill_proc(sess.get("browser_proc"))
|
| 2668 |
-
_kill_proc(sess.get("xvfb_proc"))
|
| 2669 |
-
_kill_display_processes(sess["display"])
|
| 2670 |
-
with _display_lock:
|
| 2671 |
-
_user_displays.pop(uid, None)
|
| 2672 |
-
try:
|
| 2673 |
-
_display_numbers.discard(int(sess["display"].lstrip(":")))
|
| 2674 |
-
except Exception:
|
| 2675 |
-
pass
|
| 2676 |
-
|
| 2677 |
-
|
| 2678 |
-
@app.on_event("startup")
|
| 2679 |
-
async def startup():
|
| 2680 |
-
asyncio.create_task(_cleanup_tmp())
|
| 2681 |
-
asyncio.create_task(_cleanup_idle_sessions())
|
| 2682 |
-
print("✅ Z Computer Mode v12.1 ready — CDP DOM State + Per-User Isolated Displays")
|
| 2683 |
-
print(f" Display range: :{DISPLAY_BASE} → :{DISPLAY_MAX}")
|
| 2684 |
-
print(f" Browser: {BROWSER}")
|
| 2685 |
-
print(f" CDP (Manus-style DOM): {'✅ ENABLED' if BROWSER_SUPPORTS_CDP else '⚠️ DISABLED (Firefox)'}")
|
| 2686 |
-
print(f" websockets lib: {'✅' if _CDP_AVAILABLE else '❌ (pip install websockets)'}")
|
| 2687 |
-
print(f" CDP port range: {CDP_PORT_BASE} → {CDP_PORT_BASE + (DISPLAY_MAX - DISPLAY_BASE)}")
|
| 2688 |
-
print(f" Connect: wss://your-space.hf.space/ws?user_id=EMAIL")
|
| 2689 |
-
print()
|
| 2690 |
-
print(" 🆕 New v12 actions:")
|
| 2691 |
-
print(" browser_get_elements → DOM elements list with exact coords")
|
| 2692 |
-
print(" browser_click_element → click by index id")
|
| 2693 |
-
print(" browser_type_into_element → type into input/textarea by id")
|
| 2694 |
-
print(" browser_find_element → find element by text/query")
|
| 2695 |
-
print(" browser_get_state → screenshot + DOM + text in one call")
|
| 2696 |
-
print(" browser_get_page_text → full page text content")
|
| 2697 |
-
print(" browser_execute_js → run JS in browser")
|
| 2698 |
-
print(" browser_navigate → fast CDP navigation")
|
| 2699 |
-
print(" browser_scroll_to_element → scroll to element by id")
|
| 2700 |
-
|
| 2701 |
-
|
| 2702 |
-
if __name__ == "__main__":
|
| 2703 |
-
port = int(os.environ.get("PORT", 7860))
|
| 2704 |
-
uvicorn.run("app:app", host="0.0.0.0", port=port, log_level="info")
|
|
|
|
| 1 |
"""
|
| 2 |
+
THE Z AI — Computer Mode Server v13 — CDP DOM ELEMENTS + STALENESS GUARD + COMBINED STATE
|
| 3 |
=====================================================================================
|
| 4 |
+
v13 يضيف فوق v12 تحسينات مبنية على نفس فلسفة CDP الموجودة (بدون OmniParser —
|
| 5 |
+
CDP يعطي إحداثيات حقيقية 100% من DOM مباشرة، وهو أدق وأخف من نموذج رؤية إضافي).
|
| 6 |
+
هذه التحسينات مستوحاة من فلسفة agent loop عند Manus (خطوة واحدة، ملاحظة شاملة،
|
| 7 |
+
قرار) وليست نسخة من بنيته التحتية السحابية (microVMs/E2B) التي تتطلب بنية
|
| 8 |
+
سحابية على مستوى شركة ولا يمكن تكرارها بمجرد تعديل كود.
|
| 9 |
+
|
| 10 |
+
✅ كل ما هو في v12 (CDP، browser_get_elements، browser_click_element، إلخ)
|
| 11 |
+
|
| 12 |
+
🆕 v13 — إجراءات جديدة:
|
| 13 |
+
- browser_type_into_element → يكتب مباشرة في حقل بـ index (click+type
|
| 14 |
+
في نداء واحد، مثل Manus browser_input(index, text))
|
| 15 |
+
- browser_find_element → بحث نصي عن عنصر بالوصف بدل تخمين id يدوياً
|
| 16 |
+
- browser_get_state → نداء واحد يجمع (عناصر + نص الصفحة + screenshot)
|
| 17 |
+
بدل 3 نداءات منفصلة — يقلل عدد الجولات قبل كل قرار فعلي
|
| 18 |
+
- staleness guard: click/type تتحقق أن الصفحة لم تتغيّر منذ آخر
|
| 19 |
+
browser_get_elements قبل استخدام الإحداثيات المخزّنة (يمنع نقر/كتابة
|
| 20 |
+
بإحداثيات قديمة بعد أي navigation لم يُبلَّغ عنه صراحة)
|
|
|
|
| 21 |
|
| 22 |
📋 كيف يشتغل CDP هنا:
|
| 23 |
- Chromium يُشغَّل مع --remote-debugging-port=XXXX (منفذ خاص لكل مستخدم)
|
|
|
|
| 25 |
- نقرا DOM.getDocument + DOM.querySelectorAll لكل العناصر القابلة للنقر
|
| 26 |
- نرجع قائمة JSON منظمة: [{id, tag, text, x, y, w, h, href, type}, ...]
|
| 27 |
|
| 28 |
+
🔄 التوافق الخلفي: كل أدوات v11/v12 (mouse_click, keyboard_type, إلخ) تعمل بدون تغيير
|
| 29 |
"""
|
| 30 |
|
| 31 |
import asyncio
|
|
|
|
| 456 |
x, y = new_el["x"], new_el["y"]
|
| 457 |
|
| 458 |
# النقر بالإحداثيات الحقيقية عبر xdotool (نفس محرك v11)
|
| 459 |
+
return {"ok": True, "x": x, "y": y, "tag": el.get("tag"), "text": el.get("text", "")[:50]}
|
| 460 |
|
| 461 |
|
| 462 |
+
async def cdp_type_into_element(display: str, element_id: int, elements: list, text: str,
|
| 463 |
+
clear_first: bool = True) -> dict:
|
| 464 |
+
"""
|
| 465 |
+
يكتب نصاً مباشرة داخل عنصر إدخال بـ index — مثل Manus browser_input(index, text).
|
| 466 |
+
يدمج خطوتين كانتا منفصلتين (click ثم keyboard_type) في نداء واحد، ويتحقق
|
| 467 |
+
أن العنصر فعلاً حقل إدخال قابل للكتابة قبل المحاولة (بدل الكتابة العمياء
|
| 468 |
+
في أي عنصر، وهو خطأ شائع عند تحويل click+type لخطوة واحدة).
|
| 469 |
+
يُعيد {"ok": True, "x": ..., "y": ...} أو {"error": "..."}.
|
| 470 |
+
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 471 |
el = next((e for e in elements if e.get("id") == element_id), None)
|
| 472 |
if not el:
|
| 473 |
return {"error": f"Element {element_id} not found in elements list"}
|
| 474 |
|
| 475 |
+
tag = (el.get("tag") or "").lower()
|
| 476 |
+
role = (el.get("role") or "").lower()
|
| 477 |
+
editable_tags = {"input", "textarea"}
|
| 478 |
+
editable_roles = {"textbox", "searchbox", "combobox"}
|
| 479 |
+
if tag not in editable_tags and role not in editable_roles:
|
| 480 |
+
return {
|
| 481 |
+
"error": f"Element {element_id} is a {tag or role}, not a text input. "
|
| 482 |
+
f"Use browser_click_element if you meant to click it."
|
| 483 |
+
}
|
| 484 |
|
| 485 |
+
x, y = el["x"], el["y"]
|
| 486 |
if not el.get("in_viewport", True):
|
| 487 |
+
new_data = await cdp_get_elements(display)
|
| 488 |
+
new_el = next((e for e in new_data.get("elements", []) if e.get("id") == element_id), None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
if new_el:
|
| 490 |
+
x, y = new_el["x"], new_el["y"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 491 |
|
| 492 |
+
return {"ok": True, "x": x, "y": y, "tag": tag or role, "clear_first": clear_first}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 493 |
|
| 494 |
|
| 495 |
async def cdp_get_page_text(display: str) -> dict:
|
|
|
|
| 542 |
return {"result": r.get("value"), "type": r.get("type", "undefined")}
|
| 543 |
|
| 544 |
|
| 545 |
+
async def _elements_are_stale(display: str, cached_url: str) -> bool:
|
| 546 |
+
"""
|
| 547 |
+
يتحقق أن الصفحة الحالية ما زالت نفسها اللي التُقطت منها last_elements.
|
| 548 |
+
إحداثيات عنصر مأخوذة من صفحة قديمة قد تشير الآن لعنصر مختلف تماماً
|
| 549 |
+
(أو فراغ) بعد أي navigation — هذا فحص رخيص (URL فقط) يمنع نقرات/كتابة
|
| 550 |
+
عمياء في عنصر خاطئ بعد تنقّل لم يُبلَّغ عنه صراحة.
|
| 551 |
+
"""
|
| 552 |
+
if not cached_url:
|
| 553 |
+
return False # لا معلومة سابقة كافية — لا نمنع، فقط لا نتحقق
|
| 554 |
+
current = await cdp_get_page_text(display)
|
| 555 |
+
current_url = current.get("url", "")
|
| 556 |
+
if not current_url:
|
| 557 |
+
return False # فشل الفحص نفسه — لا نمنع تنفيذ الأمر بسببه
|
| 558 |
+
return current_url != cached_url
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
def cdp_find_elements_by_text(elements: list, query: str, limit: int = 8) -> list:
|
| 562 |
+
"""
|
| 563 |
+
يبحث عن عناصر بمطابقة نصية جزئية (case-insensitive) في النص/الوصف/href.
|
| 564 |
+
يسمح للـ LLM بقول "دوس على زر تسجيل الدخول" بدل قراءة القائمة الكاملة
|
| 565 |
+
وتخمين رقم id — أقرب لطريقة تفكير Manus عن الواجهة (بالوصف الوظيفي
|
| 566 |
+
مش بمسح يدوي لكل عنصر).
|
| 567 |
+
"""
|
| 568 |
+
if not query or not elements:
|
| 569 |
+
return []
|
| 570 |
+
q = query.strip().lower()
|
| 571 |
+
scored = []
|
| 572 |
+
for el in elements:
|
| 573 |
+
haystack = " ".join([
|
| 574 |
+
str(el.get("text", "")),
|
| 575 |
+
str(el.get("href", "")),
|
| 576 |
+
str(el.get("role", "")),
|
| 577 |
+
str(el.get("type", "")),
|
| 578 |
+
]).lower()
|
| 579 |
+
if q in haystack:
|
| 580 |
+
# تفضيل التطابق الأدق (بداية النص) على التطابق الجزئي فقط
|
| 581 |
+
score = 0 if str(el.get("text", "")).lower().startswith(q) else 1
|
| 582 |
+
scored.append((score, el))
|
| 583 |
+
scored.sort(key=lambda t: t[0])
|
| 584 |
+
return [e for _, e in scored[:limit]]
|
| 585 |
+
|
| 586 |
+
|
| 587 |
async def cdp_navigate(display: str, url: str) -> dict:
|
| 588 |
"""يُنقل المتصفح لرابط معين عبر CDP — أسرع وأكثر موثوقية من xdotool."""
|
| 589 |
sess = await _get_cdp_session(display)
|
|
|
|
| 947 |
"created": time.time(),
|
| 948 |
"consecutive_capture_failures": 0, # عدّاد فشل الالتقاط المتتالي — يُصفَّر عند أي نجاح أو reset
|
| 949 |
"cdp_port": _get_cdp_port(display), # v12: منفذ CDP الخاص بهذا المستخدم
|
|
|
|
|
|
|
| 950 |
"last_elements": [], # v12: آخر قائمة عناصر DOM (للـ click_element)
|
| 951 |
}
|
| 952 |
_user_displays[user_id] = sess
|
|
|
|
| 1541 |
# ── FastAPI App ───────────────────────────────────────────────
|
| 1542 |
# ════════════════════════════════════════════════════════════════
|
| 1543 |
|
| 1544 |
+
app = FastAPI(title="Z-Computer-Mode v11 — Per-User Isolation")
|
| 1545 |
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True,
|
| 1546 |
allow_methods=["*"], allow_headers=["*"])
|
| 1547 |
|
|
|
|
| 1612 |
lines.append(_format_single_element(el))
|
| 1613 |
|
| 1614 |
lines.append(f"\n💡 To click element N: use action='browser_click_element' with id=N")
|
| 1615 |
+
lines.append(f"💡 To type into an input N: use action='browser_type_into_element' with id=N, text='...'")
|
| 1616 |
+
lines.append(f"💡 To find an element by description: use action='browser_find_element' with query='...'")
|
| 1617 |
return "\n".join(lines)
|
| 1618 |
|
| 1619 |
|
|
|
|
| 2215 |
result = await cdp_get_elements(display)
|
| 2216 |
elements = result.get("elements", [])
|
| 2217 |
|
| 2218 |
+
# خزّن قائمة العناصر + رابط الصفحة وقت الالتقاط، للاستخدام في
|
| 2219 |
+
# click/type/scroll لاحقاً مع التحقق من عدم تغيّر الصفحة (staleness guard)
|
| 2220 |
with _display_lock:
|
| 2221 |
if sess["user_id"] in _user_displays:
|
| 2222 |
_user_displays[sess["user_id"]]["last_elements"] = elements
|
| 2223 |
+
_user_displays[sess["user_id"]]["last_elements_url"] = result.get("page_url", "")
|
|
|
|
| 2224 |
|
| 2225 |
await send({
|
| 2226 |
"type": "browser_elements",
|
|
|
|
| 2244 |
element_id = int(data.get("id", -1))
|
| 2245 |
|
| 2246 |
with _display_lock:
|
| 2247 |
+
user_disp = _user_displays.get(sess["user_id"], {})
|
| 2248 |
+
elements = user_disp.get("last_elements", [])
|
| 2249 |
+
cached_url = user_disp.get("last_elements_url", "")
|
| 2250 |
|
| 2251 |
if element_id < 0 or not elements:
|
| 2252 |
await send({
|
|
|
|
| 2255 |
})
|
| 2256 |
return
|
| 2257 |
|
| 2258 |
+
if await _elements_are_stale(display, cached_url):
|
| 2259 |
+
await send({
|
| 2260 |
+
"type": "error",
|
| 2261 |
+
"msg": "Page changed since last browser_get_elements — element coordinates are stale. Call browser_get_elements again before clicking.",
|
| 2262 |
+
"stale": True,
|
| 2263 |
+
})
|
| 2264 |
return
|
| 2265 |
|
| 2266 |
click_info = await cdp_click_element(display, element_id, elements)
|
|
|
|
| 2285 |
asyncio.create_task(shot_bg("after element click", delay=0.8, extra_shot_delay=2.0))
|
| 2286 |
|
| 2287 |
# ── browser_type_into_element ────────────────────────────
|
| 2288 |
+
# يكتب مباشرة في حقل بـ index — يدمج click + keyboard_type في نداء
|
| 2289 |
+
# واحد (مثل Manus browser_input(index, text)) بدل خطوتين منفصلتين
|
| 2290 |
elif action == "browser_type_into_element":
|
| 2291 |
element_id = int(data.get("id", -1))
|
| 2292 |
+
text = data.get("text", "")
|
| 2293 |
+
clear_first = data.get("clear_first", True)
|
| 2294 |
+
|
| 2295 |
with _display_lock:
|
| 2296 |
+
user_disp = _user_displays.get(sess["user_id"], {})
|
| 2297 |
+
elements = user_disp.get("last_elements", [])
|
| 2298 |
+
cached_url = user_disp.get("last_elements_url", "")
|
| 2299 |
+
|
| 2300 |
if element_id < 0 or not elements:
|
| 2301 |
+
await send({
|
| 2302 |
+
"type": "error",
|
| 2303 |
+
"msg": "Call browser_get_elements first, then use the returned element id",
|
| 2304 |
+
})
|
| 2305 |
return
|
| 2306 |
+
|
| 2307 |
+
if await _elements_are_stale(display, cached_url):
|
| 2308 |
+
await send({
|
| 2309 |
+
"type": "error",
|
| 2310 |
+
"msg": "Page changed since last browser_get_elements — element coordinates are stale. Call browser_get_elements again before typing.",
|
| 2311 |
+
"stale": True,
|
| 2312 |
+
})
|
| 2313 |
return
|
| 2314 |
+
|
| 2315 |
+
type_info = await cdp_type_into_element(display, element_id, elements, text, clear_first)
|
| 2316 |
if "error" in type_info:
|
| 2317 |
await send({"type": "error", "msg": type_info["error"]})
|
| 2318 |
return
|
| 2319 |
+
|
| 2320 |
+
x, y = type_info["x"], type_info["y"]
|
| 2321 |
+
await xdo(["mousemove", "--sync", str(x), str(y)], display)
|
| 2322 |
+
await asyncio.sleep(0.07)
|
| 2323 |
await xdo(["click", "1"], display)
|
| 2324 |
+
await asyncio.sleep(0.05)
|
| 2325 |
+
if clear_first:
|
| 2326 |
+
await xdo(["key", "--clearmodifiers", "ctrl+a"], display)
|
| 2327 |
+
await xdo(["key", "--clearmodifiers", "Delete"], display)
|
| 2328 |
+
await type_smart(text, display)
|
| 2329 |
+
|
| 2330 |
await send({
|
| 2331 |
"type": "ack",
|
| 2332 |
"action": "browser_type_into_element",
|
| 2333 |
"element_id": element_id,
|
| 2334 |
+
"x": x, "y": y,
|
| 2335 |
+
"text_len": len(text),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2336 |
})
|
| 2337 |
+
asyncio.create_task(shot_
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|