prithivMLmods commited on
Commit
d35e035
·
verified ·
1 Parent(s): 342fdab

update app

Browse files
Files changed (1) hide show
  1. app.py +1359 -370
app.py CHANGED
@@ -1,11 +1,12 @@
1
  import os
2
  import re
 
3
  import json
4
  import time
5
- import unicodedata
6
- import gc
7
  from io import BytesIO
8
- from typing import Iterable, Tuple, Optional, List, Dict, Any
 
9
 
10
  import gradio as gr
11
  import numpy as np
@@ -17,94 +18,32 @@ from transformers import (
17
  Qwen2_5_VLForConditionalGeneration,
18
  AutoProcessor,
19
  AutoModelForImageTextToText,
20
- AutoTokenizer,
21
- AutoModelForVision2Seq
22
  )
23
  from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
24
  from qwen_vl_utils import process_vision_info
25
 
26
- from gradio.themes import Soft
27
- from gradio.themes.utils import colors, fonts, sizes
28
-
29
- colors.orange_red = colors.Color(
30
- name="orange_red",
31
- c50="#FFF0E5",
32
- c100="#FFE0CC",
33
- c200="#FFC299",
34
- c300="#FFA366",
35
- c400="#FF8533",
36
- c500="#FF4500",
37
- c600="#E63E00",
38
- c700="#CC3700",
39
- c800="#B33000",
40
- c900="#992900",
41
- c950="#802200",
42
- )
43
-
44
- class OrangeRedTheme(Soft):
45
- def __init__(
46
- self,
47
- *,
48
- primary_hue: colors.Color | str = colors.gray,
49
- secondary_hue: colors.Color | str = colors.orange_red,
50
- neutral_hue: colors.Color | str = colors.slate,
51
- text_size: sizes.Size | str = sizes.text_lg,
52
- font: fonts.Font | str | Iterable[fonts.Font | str] = (
53
- fonts.GoogleFont("Outfit"), "Arial", "sans-serif",
54
- ),
55
- font_mono: fonts.Font | str | Iterable[fonts.Font | str] = (
56
- fonts.GoogleFont("IBM Plex Mono"), "ui-monospace", "monospace",
57
- ),
58
- ):
59
- super().__init__(
60
- primary_hue=primary_hue,
61
- secondary_hue=secondary_hue,
62
- neutral_hue=neutral_hue,
63
- text_size=text_size,
64
- font=font,
65
- font_mono=font_mono,
66
- )
67
- super().set(
68
- background_fill_primary="*primary_50",
69
- background_fill_primary_dark="*primary_900",
70
- body_background_fill="linear-gradient(135deg, *primary_200, *primary_100)",
71
- body_background_fill_dark="linear-gradient(135deg, *primary_900, *primary_800)",
72
- button_primary_text_color="white",
73
- button_primary_text_color_hover="white",
74
- button_primary_background_fill="linear-gradient(90deg, *secondary_500, *secondary_600)",
75
- button_primary_background_fill_hover="linear-gradient(90deg, *secondary_600, *secondary_700)",
76
- button_primary_background_fill_dark="linear-gradient(90deg, *secondary_600, *secondary_700)",
77
- button_primary_background_fill_hover_dark="linear-gradient(90deg, *secondary_500, *secondary_600)",
78
- button_secondary_text_color="black",
79
- button_secondary_text_color_hover="white",
80
- button_secondary_background_fill="linear-gradient(90deg, *primary_300, *primary_300)",
81
- button_secondary_background_fill_hover="linear-gradient(90deg, *primary_400, *primary_400)",
82
- button_secondary_background_fill_dark="linear-gradient(90deg, *primary_500, *primary_600)",
83
- button_secondary_background_fill_hover_dark="linear-gradient(90deg, *primary_500, *primary_500)",
84
- slider_color="*secondary_500",
85
- slider_color_dark="*secondary_600",
86
- block_title_text_weight="600",
87
- block_border_width="3px",
88
- block_shadow="*shadow_drop_lg",
89
- button_primary_shadow="*shadow_drop_lg",
90
- button_large_padding="11px",
91
- color_accent_soft="*primary_100",
92
- block_label_background_fill="*primary_200",
93
- )
94
 
95
- orange_red_theme = OrangeRedTheme()
96
-
97
- device = "cuda" if torch.cuda.is_available() else "cpu"
98
- print(f"Running on device: {device}")
 
 
 
 
99
 
100
  print("🔄 Loading Fara-7B...")
101
- MODEL_ID_V = "microsoft/Fara-7B"
102
  try:
103
  processor_v = AutoProcessor.from_pretrained(MODEL_ID_V, trust_remote_code=True)
104
  model_v = Qwen2_5_VLForConditionalGeneration.from_pretrained(
105
  MODEL_ID_V,
106
  trust_remote_code=True,
107
- torch_dtype=torch.float16
108
  ).to(device).eval()
109
  except Exception as e:
110
  print(f"Failed to load Fara: {e}")
@@ -118,7 +57,7 @@ try:
118
  model_x = AutoModelForImageTextToText.from_pretrained(
119
  MODEL_ID_X,
120
  trust_remote_code=True,
121
- torch_dtype=torch.bfloat16 if device == "cuda" else torch.float32,
122
  ).to(device).eval()
123
  except Exception as e:
124
  print(f"Failed to load UI-TARS: {e}")
@@ -126,13 +65,13 @@ except Exception as e:
126
  processor_x = None
127
 
128
  print("🔄 Loading Holo2-4B...")
129
- MODEL_ID_H = "Hcompany/Holo2-4B"
130
  try:
131
  processor_h = AutoProcessor.from_pretrained(MODEL_ID_H, trust_remote_code=True)
132
  model_h = AutoModelForImageTextToText.from_pretrained(
133
  MODEL_ID_H,
134
  trust_remote_code=True,
135
- torch_dtype=torch.bfloat16 if device == "cuda" else torch.float32,
136
  ).to(device).eval()
137
  except Exception as e:
138
  print(f"Failed to load Holo2: {e}")
@@ -142,13 +81,12 @@ except Exception as e:
142
  print("🔄 Loading ActIO-UI-7B...")
143
  MODEL_ID_ACT = "Uniphore/actio-ui-7b-rlvr"
144
  try:
145
- # ActIO usually relies on Qwen2VL architecture structure
146
  processor_act = AutoProcessor.from_pretrained(MODEL_ID_ACT, trust_remote_code=True)
147
  model_act = AutoModelForVision2Seq.from_pretrained(
148
  MODEL_ID_ACT,
149
  trust_remote_code=True,
150
- torch_dtype=torch.float16 if device == "cuda" else torch.float32,
151
- device_map=None # We will move to device manually to control memory
152
  ).to(device).eval()
153
  except Exception as e:
154
  print(f"Failed to load ActIO-UI: {e}")
@@ -157,22 +95,115 @@ except Exception as e:
157
 
158
  print("✅ Models loading sequence complete.")
159
 
160
- def array_to_image(image_array: np.ndarray) -> Image.Image:
161
- if image_array is None: raise ValueError("No image provided.")
162
- return Image.fromarray(np.uint8(image_array))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
163
 
164
  def get_image_proc_params(processor) -> Dict[str, int]:
165
  ip = getattr(processor, "image_processor", None)
166
-
167
  default_min = 256 * 256
168
  default_max = 1280 * 1280
169
-
170
  patch_size = getattr(ip, "patch_size", 14)
171
  merge_size = getattr(ip, "merge_size", 2)
172
  min_pixels = getattr(ip, "min_pixels", default_min)
173
  max_pixels = getattr(ip, "max_pixels", default_max)
174
 
175
- # Holo2/Qwen specific sizing sometimes in 'size' dict
176
  size_config = getattr(ip, "size", {})
177
  if isinstance(size_config, dict):
178
  if "shortest_edge" in size_config:
@@ -180,8 +211,10 @@ def get_image_proc_params(processor) -> Dict[str, int]:
180
  if "longest_edge" in size_config:
181
  max_pixels = size_config["longest_edge"]
182
 
183
- if min_pixels is None: min_pixels = default_min
184
- if max_pixels is None: max_pixels = default_max
 
 
185
 
186
  return {
187
  "patch_size": patch_size,
@@ -191,18 +224,16 @@ def get_image_proc_params(processor) -> Dict[str, int]:
191
  }
192
 
193
  def apply_chat_template_compat(processor, messages: List[Dict[str, Any]], thinking: bool = True) -> str:
194
- # Holo2 specific: allows turning thinking off in template
195
  if hasattr(processor, "apply_chat_template"):
196
  try:
197
  return processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, thinking=thinking)
198
  except TypeError:
199
- # Fallback for processors that don't support 'thinking' kwarg
200
  return processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
201
-
202
  tok = getattr(processor, "tokenizer", None)
203
  if tok is not None and hasattr(tok, "apply_chat_template"):
204
  return tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
205
-
206
  raise AttributeError("Could not apply chat template.")
207
 
208
  def trim_generated(generated_ids, inputs):
@@ -215,13 +246,13 @@ def trim_generated(generated_ids, inputs):
215
 
216
  def get_fara_prompt(task, image):
217
  OS_SYSTEM_PROMPT = """You are a GUI agent. You are given a task and a screenshot of the current status.
218
- You need to generate the next action to complete the task.
219
- Output your action inside a <tool_call> block using JSON format.
220
- Include "coordinate": [x, y] in pixels for interactions.
221
- Examples:
222
- <tool_call>{"name": "User", "arguments": {"action": "click", "coordinate": [400, 300]}}</tool_call>
223
- <tool_call>{"name": "User", "arguments": {"action": "type", "coordinate": [100, 200], "text": "hello"}}</tool_call>
224
- """
225
  return [
226
  {"role": "system", "content": [{"type": "text", "text": OS_SYSTEM_PROMPT}]},
227
  {"role": "user", "content": [{"type": "image", "image": image}, {"type": "text", "text": f"Instruction: {task}"}]},
@@ -233,32 +264,26 @@ def get_localization_prompt(task, image):
233
  "output a click position as Click(x, y) with x num pixels from the left edge "
234
  "and y num pixels from the top edge."
235
  )
236
- return [
237
- {
238
- "role": "user",
239
- "content": [
240
- {"type": "image", "image": image},
241
- {"type": "text", "text": f"{guidelines}\n{task}"}
242
- ],
243
- }
244
- ]
245
 
246
  def get_holo2_prompt(task, image):
247
  schema_str = '{"properties": {"x": {"description": "The x coordinate, normalized between 0 and 1000.", "ge": 0, "le": 1000, "title": "X", "type": "integer"}, "y": {"description": "The y coordinate, normalized between 0 and 1000.", "ge": 0, "le": 1000, "title": "Y", "type": "integer"}}, "required": ["x", "y"], "title": "ClickCoordinates", "type": "object"}'
248
-
249
  prompt = f"""Localize an element on the GUI image according to the provided target and output a click position.
250
- * You must output a valid JSON following the format: {schema_str}
251
- Your target is:"""
252
-
253
- return [
254
- {
255
- "role": "user",
256
- "content": [
257
- {"type": "image", "image": image},
258
- {"type": "text", "text": f"{prompt}\n{task}"},
259
- ],
260
- },
261
- ]
262
 
263
  def get_actio_prompt(task, image):
264
  system_prompt = (
@@ -283,9 +308,7 @@ def get_actio_prompt(task, image):
283
  def parse_click_response(text: str) -> List[Dict]:
284
  actions = []
285
  text = text.strip()
286
-
287
- # Generic Point parsing (ActIO uses similar click(x,y) format often)
288
- # Looking for Click(x, y), left_click(x, y), etc.
289
  matches_click = re.findall(r"(?:click|left_click|right_click|double_click)\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)", text, re.IGNORECASE)
290
  for m in matches_click:
291
  actions.append({"type": "click", "x": int(m[0]), "y": int(m[1]), "text": "", "norm": False})
@@ -297,8 +320,7 @@ def parse_click_response(text: str) -> List[Dict]:
297
  matches_box = re.findall(r"start_box=['\"]?\(\s*(\d+)\s*,\s*(\d+)\s*\)['\"]?", text, re.IGNORECASE)
298
  for m in matches_box:
299
  actions.append({"type": "click", "x": int(m[0]), "y": int(m[1]), "text": "", "norm": False})
300
-
301
- # Fallback tuple
302
  if not actions:
303
  matches_tuple = re.findall(r"(?:^|\s)\(\s*(\d+)\s*,\s*(\d+)\s*\)(?:$|\s|,)", text)
304
  for m in matches_tuple:
@@ -322,36 +344,31 @@ def parse_fara_response(response: str) -> List[Dict]:
322
  })
323
  except Exception as e:
324
  print(f"Error parsing Fara JSON: {e}")
325
- pass
326
  return actions
327
 
328
  def parse_holo2_response(response: str) -> List[Dict]:
329
  actions = []
330
  try:
331
  data = json.loads(response.strip())
332
- if 'x' in data and 'y' in data:
333
- actions.append({"type": "click", "x": int(data['x']), "y": int(data['y']), "text": "*", "norm": True})
334
  return actions
335
- except:
336
  pass
337
 
338
  match = re.search(r"\{\s*['\"]x['\"]\s*:\s*(\d+)\s*,\s*['\"]y['\"]\s*:\s*(\d+)\s*\}", response)
339
  if match:
340
  actions.append({
341
- "type": "click",
342
- "x": int(match.group(1)),
343
- "y": int(match.group(2)),
344
- "text": "Holo2",
345
- "norm": True
346
  })
347
- return actions
348
  return actions
349
 
350
  def parse_actio_response(response: str) -> List[Dict]:
351
- # Expected format: <action>(x, y) e.g., click(551, 355)
352
- # It might also just output "click(551, 355)" or "left_click(551, 355)"
353
  actions = []
354
- # General regex for name(x, y)
355
  matches = re.findall(r"([a-zA-Z_]+)\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)", response)
356
  for action_name, x, y in matches:
357
  actions.append({
@@ -359,268 +376,1240 @@ def parse_actio_response(response: str) -> List[Dict]:
359
  "x": int(x),
360
  "y": int(y),
361
  "text": "",
362
- "norm": False # ActIO usually outputs absolute coordinates relative to input image
363
  })
364
  return actions
365
 
366
- def create_localized_image(original_image: Image.Image, actions: list[dict]) -> Optional[Image.Image]:
367
- if not actions: return None
 
 
368
  img_copy = original_image.copy()
369
  draw = ImageDraw.Draw(img_copy)
370
-
371
  try:
372
  font = ImageFont.load_default(size=18)
373
- except IOError:
374
  font = ImageFont.load_default()
375
-
376
  for act in actions:
377
- x = act['x']
378
- y = act['y']
379
-
380
- pixel_x, pixel_y = int(x), int(y)
381
-
382
- color = 'red' if 'click' in act['type'].lower() else 'blue'
383
-
384
- # Draw Crosshair
385
  line_len = 15
386
  width = 4
387
- # Horizontal
388
- draw.line((pixel_x - line_len, pixel_y, pixel_x + line_len, pixel_y), fill=color, width=width)
389
- # Vertical
390
- draw.line((pixel_x, pixel_y - line_len, pixel_x, pixel_y + line_len), fill=color, width=width)
391
-
392
- # Outer Circle
393
  r = 20
394
- draw.ellipse([pixel_x - r, pixel_y - r, pixel_x + r, pixel_y + r], outline=color, width=3)
395
-
396
  label = f"{act['type']}"
397
- if act.get('text'): label += f": \"{act['text']}\""
398
-
399
- text_pos = (pixel_x + 25, pixel_y - 15)
400
-
401
- # Label with background
402
  try:
403
  bbox = draw.textbbox(text_pos, label, font=font)
404
- padded_bbox = (bbox[0]-4, bbox[1]-2, bbox[2]+4, bbox[3]+2)
405
  draw.rectangle(padded_bbox, fill="yellow", outline=color)
406
  draw.text(text_pos, label, fill="black", font=font)
407
- except Exception as e:
408
- draw.text(text_pos, label, fill="white")
409
 
410
  return img_copy
411
 
412
- @spaces.GPU
413
- def process_screenshot(input_numpy_image: np.ndarray, task: str, model_choice: str):
414
- if input_numpy_image is None: return "⚠️ Please upload an image.", None
415
- if not task.strip(): return "⚠️ Please provide a task instruction.", None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
416
 
417
- input_pil_image = array_to_image(input_numpy_image)
418
- orig_w, orig_h = input_pil_image.size
419
- actions = []
420
- raw_response = ""
421
-
422
- if model_choice == "Fara-7B":
423
- if model_v is None: return "Error: Fara model failed to load.", None
424
- print("Using Fara Pipeline...")
425
-
426
- messages = get_fara_prompt(task, input_pil_image)
427
- text_prompt = processor_v.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
428
- image_inputs, video_inputs = process_vision_info(messages)
429
-
430
- inputs = processor_v(
431
- text=[text_prompt],
432
- images=image_inputs,
433
- videos=video_inputs,
434
- padding=True,
435
- return_tensors="pt"
436
- )
437
- inputs = inputs.to(device)
438
-
439
- with torch.no_grad():
440
- generated_ids = model_v.generate(**inputs, max_new_tokens=512)
441
-
442
- generated_ids = trim_generated(generated_ids, inputs)
443
- raw_response = processor_v.batch_decode(generated_ids, skip_special_tokens=True)[0]
444
- actions = parse_fara_response(raw_response)
445
-
446
- elif model_choice == "Holo2-4B":
447
- if model_h is None: return "Error: Holo2 model failed to load.", None
448
- print("Using Holo2-4B Pipeline...")
449
-
450
- model, processor = model_h, processor_h
451
- ip_params = get_image_proc_params(processor)
452
-
453
- resized_h, resized_w = smart_resize(
454
- input_pil_image.height, input_pil_image.width,
455
- factor=ip_params["patch_size"] * ip_params["merge_size"],
456
- min_pixels=ip_params["min_pixels"],
457
- max_pixels=ip_params["max_pixels"]
458
- )
459
- proc_image = input_pil_image.resize((resized_w, resized_h), Image.Resampling.LANCZOS)
460
-
461
- messages = get_holo2_prompt(task, proc_image)
462
- text_prompt = apply_chat_template_compat(processor, messages, thinking=False)
463
-
464
- inputs = processor(text=[text_prompt], images=[proc_image], padding=True, return_tensors="pt")
465
- inputs = {k: v.to(device) for k, v in inputs.items()}
466
-
467
- with torch.no_grad():
468
- generated_ids = model.generate(**inputs, max_new_tokens=128)
469
-
470
- generated_ids = trim_generated(generated_ids, inputs)
471
- raw_response = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
472
- actions = parse_holo2_response(raw_response)
473
-
474
- # Scale Holo2 coordinates (Normalized 0-1000 -> Original Pixel)
475
- for a in actions:
476
- if a.get('norm', False):
477
- a['x'] = (a['x'] / 1000.0) * orig_w
478
- a['y'] = (a['y'] / 1000.0) * orig_h
479
-
480
- elif model_choice == "UI-TARS-1.5-7B":
481
- if model_x is None: return "Error: UI-TARS model failed to load.", None
482
- print("Using UI-TARS Pipeline...")
483
-
484
- model, processor = model_x, processor_x
485
- ip_params = get_image_proc_params(processor)
486
-
487
- resized_h, resized_w = smart_resize(
488
- input_pil_image.height, input_pil_image.width,
489
- factor=ip_params["patch_size"] * ip_params["merge_size"],
490
- min_pixels=ip_params["min_pixels"],
491
- max_pixels=ip_params["max_pixels"]
492
- )
493
- proc_image = input_pil_image.resize((resized_w, resized_h), Image.Resampling.LANCZOS)
494
-
495
- messages = get_localization_prompt(task, proc_image)
496
- text_prompt = apply_chat_template_compat(processor, messages)
497
-
498
- inputs = processor(text=[text_prompt], images=[proc_image], padding=True, return_tensors="pt")
499
- inputs = {k: v.to(device) for k, v in inputs.items()}
500
-
501
- with torch.no_grad():
502
- generated_ids = model.generate(**inputs, max_new_tokens=128)
503
-
504
- generated_ids = trim_generated(generated_ids, inputs)
505
- raw_response = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
506
- actions = parse_click_response(raw_response)
507
-
508
- # Scale UI-TARS coordinates (Resized Pixel -> Original Pixel)
509
- if resized_w > 0 and resized_h > 0:
510
- scale_x = orig_w / resized_w
511
- scale_y = orig_h / resized_h
512
  for a in actions:
513
- a['x'] = int(a['x'] * scale_x)
514
- a['y'] = int(a['y'] * scale_y)
515
-
516
- elif model_choice == "ActIO-UI-7B":
517
- if model_act is None: return "Error: ActIO model failed to load.", None
518
- print("Using ActIO-UI Pipeline...")
519
-
520
- model, processor = model_act, processor_act
521
-
522
- # ActIO generally uses Qwen2-VL like processing
523
- # We need to construct the prompt with text and image
524
- messages = get_actio_prompt(task, input_pil_image)
525
-
526
- text_prompt = processor.apply_chat_template(
527
- messages, tokenize=False, add_generation_prompt=True
528
- )
529
 
530
- # ActIO typically works with standard RGB images
531
- inputs = processor(
532
- text=[text_prompt],
533
- images=[input_pil_image],
534
- padding=True,
535
- return_tensors="pt"
536
- )
537
- inputs = {k: v.to(device) for k, v in inputs.items()}
538
 
539
- with torch.no_grad():
540
- generated_ids = model.generate(
541
- **inputs,
542
- max_new_tokens=1024, # ActIO allows verbose output sometimes
543
- do_sample=False,
 
 
544
  )
 
545
 
546
- generated_ids = trim_generated(generated_ids, inputs)
547
- raw_response = processor.batch_decode(
548
- generated_ids,
549
- skip_special_tokens=True,
550
- clean_up_tokenization_spaces=False
551
- )[0]
552
-
553
- actions = parse_actio_response(raw_response)
554
-
555
- # ActIO usually outputs absolute coordinates based on the input image resolution provided to the processor.
556
- # Since we passed the original PIL image (unless resized internally by processor to something widely different),
557
- # these coords are usually correct. If ActIO resizes internally and outputs coords relative to resize,
558
- # we might need scaling, but standard usage implies absolute.
559
- pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
560
 
561
- else:
562
- return f"Error: Unknown model '{model_choice}'", None
 
 
 
 
 
 
 
 
 
 
 
563
 
564
- print(f"Raw Output: {raw_response}")
565
- print(f"Parsed Actions: {actions}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
566
 
567
- output_image = input_pil_image
568
- if actions:
569
- vis = create_localized_image(input_pil_image, actions)
570
- if vis: output_image = vis
571
-
572
- return raw_response, output_image
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
573
 
574
- css="""
575
- #col-container {
576
- margin: 0 auto;
577
- max-width: 960px;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
578
  }
579
- #main-title h1 {font-size: 2.1em !important;}
580
  """
 
581
  with gr.Blocks() as demo:
582
- gr.Markdown("# **CUA GUI Operator 🖥️**", elem_id="main-title")
583
- gr.Markdown("Perform Computer Use Agent tasks with the models: [Fara-7B](https://huggingface.co/microsoft/Fara-7B), [UI-TARS-1.5-7B](https://huggingface.co/ByteDance-Seed/UI-TARS-1.5-7B), [Holo2-4B](https://huggingface.co/Hcompany/Holo2-4B), and [ActIO-UI-7B](https://huggingface.co/Uniphore/actio-ui-7b-rlvr).")
584
-
585
- with gr.Row():
586
- with gr.Column(scale=2):
587
- input_image = gr.Image(label="Upload UI Image", type="numpy", height=500)
588
-
589
- with gr.Row():
590
- model_choice = gr.Radio(
591
- choices=["Fara-7B", "UI-TARS-1.5-7B", "Holo2-4B", "ActIO-UI-7B"],
592
- label="Select Model",
593
- value="Fara-7B",
594
- interactive=True
595
- )
596
-
597
- task_input = gr.Textbox(
598
- label="Task Instruction",
599
- placeholder="e.g. Click on the search bar",
600
- lines=2
601
- )
602
- submit_btn = gr.Button("Call CUA Agent", variant="primary")
603
 
604
- with gr.Column(scale=3):
605
- output_image = gr.Image(label="Visualized Action Points", elem_id="out_img", height=500)
606
- output_text = gr.Textbox(label="Agent Model Response", lines=10)
607
 
608
- submit_btn.click(
609
- fn=process_screenshot,
610
- inputs=[input_image, task_input, model_choice],
611
- outputs=[output_text, output_image]
612
- )
613
-
614
- gr.Examples(
615
- examples=[
616
- ["examples/1.png", "Click on the Fara-7B model.", "Fara-7B"],
617
- ["examples/2.png", "Click on the VLMs Collection", "UI-TARS-1.5-7B"],
618
- ["examples/3.png", "Click on the 'SAM3'.", "Holo2-4B"],
619
- ["examples/1.png", "Click on the Fara-7B model.", "ActIO-UI-7B"],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
620
  ],
621
- inputs=[input_image, task_input, model_choice],
622
- label="Quick Examples"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
623
  )
624
 
625
  if __name__ == "__main__":
626
- demo.queue(max_size=50).launch(theme=orange_red_theme, css=css, mcp_server=True, ssr_mode=False, show_error=True)
 
 
 
 
 
 
 
1
  import os
2
  import re
3
+ import gc
4
  import json
5
  import time
6
+ import base64
 
7
  from io import BytesIO
8
+ from threading import Thread
9
+ from typing import List, Dict, Any, Optional
10
 
11
  import gradio as gr
12
  import numpy as np
 
18
  Qwen2_5_VLForConditionalGeneration,
19
  AutoProcessor,
20
  AutoModelForImageTextToText,
21
+ AutoModelForVision2Seq,
 
22
  )
23
  from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
24
  from qwen_vl_utils import process_vision_info
25
 
26
+ ACCENT = "#FFFF00"
27
+ MAX_INPUT_TEXT_LENGTH = int(os.getenv("MAX_INPUT_TEXT_LENGTH", "2048"))
28
+ device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
 
30
+ print("Running on device:", device)
31
+ print("torch.__version__ =", torch.__version__)
32
+ print("torch.version.cuda =", torch.version.cuda)
33
+ print("cuda available:", torch.cuda.is_available())
34
+ print("cuda device count:", torch.cuda.device_count())
35
+ if torch.cuda.is_available():
36
+ print("current device:", torch.cuda.current_device())
37
+ print("device name:", torch.cuda.get_device_name(torch.cuda.current_device()))
38
 
39
  print("🔄 Loading Fara-7B...")
40
+ MODEL_ID_V = "microsoft/Fara-7B"
41
  try:
42
  processor_v = AutoProcessor.from_pretrained(MODEL_ID_V, trust_remote_code=True)
43
  model_v = Qwen2_5_VLForConditionalGeneration.from_pretrained(
44
  MODEL_ID_V,
45
  trust_remote_code=True,
46
+ torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
47
  ).to(device).eval()
48
  except Exception as e:
49
  print(f"Failed to load Fara: {e}")
 
57
  model_x = AutoModelForImageTextToText.from_pretrained(
58
  MODEL_ID_X,
59
  trust_remote_code=True,
60
+ torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
61
  ).to(device).eval()
62
  except Exception as e:
63
  print(f"Failed to load UI-TARS: {e}")
 
65
  processor_x = None
66
 
67
  print("🔄 Loading Holo2-4B...")
68
+ MODEL_ID_H = "Hcompany/Holo2-4B"
69
  try:
70
  processor_h = AutoProcessor.from_pretrained(MODEL_ID_H, trust_remote_code=True)
71
  model_h = AutoModelForImageTextToText.from_pretrained(
72
  MODEL_ID_H,
73
  trust_remote_code=True,
74
+ torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
75
  ).to(device).eval()
76
  except Exception as e:
77
  print(f"Failed to load Holo2: {e}")
 
81
  print("🔄 Loading ActIO-UI-7B...")
82
  MODEL_ID_ACT = "Uniphore/actio-ui-7b-rlvr"
83
  try:
 
84
  processor_act = AutoProcessor.from_pretrained(MODEL_ID_ACT, trust_remote_code=True)
85
  model_act = AutoModelForVision2Seq.from_pretrained(
86
  MODEL_ID_ACT,
87
  trust_remote_code=True,
88
+ torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
89
+ device_map=None
90
  ).to(device).eval()
91
  except Exception as e:
92
  print(f"Failed to load ActIO-UI: {e}")
 
95
 
96
  print("✅ Models loading sequence complete.")
97
 
98
+ MODEL_MAP = {
99
+ "Fara-7B": (processor_v, model_v),
100
+ "UI-TARS-1.5-7B": (processor_x, model_x),
101
+ "Holo2-4B": (processor_h, model_h),
102
+ "ActIO-UI-7B": (processor_act, model_act),
103
+ }
104
+ MODEL_CHOICES = list(MODEL_MAP.keys())
105
+
106
+ image_examples = [
107
+ {"query": "Click on the Fara-7B model.", "image": "examples/1.png", "model": "Fara-7B"},
108
+ {"query": "Click on the VLMs Collection", "image": "examples/2.png", "model": "UI-TARS-1.5-7B"},
109
+ {"query": "Click on the 'SAM3'.", "image": "examples/3.png", "model": "Holo2-4B"},
110
+ {"query": "Click on the Fara-7B model.", "image": "examples/1.png", "model": "ActIO-UI-7B"},
111
+ ]
112
+
113
+ def pil_to_data_url(img: Image.Image, fmt="PNG"):
114
+ buf = BytesIO()
115
+ img.save(buf, format=fmt)
116
+ data = base64.b64encode(buf.getvalue()).decode()
117
+ mime = "image/png" if fmt.upper() == "PNG" else "image/jpeg"
118
+ return f"data:{mime};base64,{data}"
119
+
120
+ def file_to_data_url(path):
121
+ if not os.path.exists(path):
122
+ return ""
123
+ ext = path.rsplit(".", 1)[-1].lower()
124
+ mime = {
125
+ "jpg": "image/jpeg",
126
+ "jpeg": "image/jpeg",
127
+ "png": "image/png",
128
+ "webp": "image/webp",
129
+ }.get(ext, "image/jpeg")
130
+ with open(path, "rb") as f:
131
+ data = base64.b64encode(f.read()).decode()
132
+ return f"data:{mime};base64,{data}"
133
+
134
+ def make_thumb_b64(path, max_dim=240):
135
+ try:
136
+ img = Image.open(path).convert("RGB")
137
+ img.thumbnail((max_dim, max_dim))
138
+ return pil_to_data_url(img, "JPEG")
139
+ except Exception as e:
140
+ print("Thumbnail error:", e)
141
+ return ""
142
+
143
+ def b64_to_pil(b64_str):
144
+ if not b64_str:
145
+ return None
146
+ try:
147
+ if b64_str.startswith("data:"):
148
+ _, data = b64_str.split(",", 1)
149
+ else:
150
+ data = b64_str
151
+ image_data = base64.b64decode(data)
152
+ return Image.open(BytesIO(image_data)).convert("RGB")
153
+ except Exception:
154
+ return None
155
+
156
+ def build_example_cards_html():
157
+ cards = ""
158
+ for i, ex in enumerate(image_examples):
159
+ thumb = make_thumb_b64(ex["image"])
160
+ prompt_short = ex["query"][:72] + ("..." if len(ex["query"]) > 72 else "")
161
+ cards += f"""
162
+ <div class="example-card" data-idx="{i}">
163
+ <div class="example-thumb-wrap">
164
+ {"<img src='" + thumb + "' alt=''>" if thumb else "<div class='example-thumb-placeholder'>Preview</div>"}
165
+ </div>
166
+ <div class="example-meta-row">
167
+ <span class="example-badge">{ex["model"]}</span>
168
+ </div>
169
+ <div class="example-prompt-text">{prompt_short}</div>
170
+ </div>
171
+ """
172
+ return cards
173
+
174
+ EXAMPLE_CARDS_HTML = build_example_cards_html()
175
+
176
+ def load_example_data(idx_str):
177
+ try:
178
+ idx = int(str(idx_str).strip())
179
+ except Exception:
180
+ return gr.update(value=json.dumps({"status": "error", "message": "Invalid example index"}))
181
+
182
+ if idx < 0 or idx >= len(image_examples):
183
+ return gr.update(value=json.dumps({"status": "error", "message": "Example index out of range"}))
184
+
185
+ ex = image_examples[idx]
186
+ img_b64 = file_to_data_url(ex["image"])
187
+ if not img_b64:
188
+ return gr.update(value=json.dumps({"status": "error", "message": "Could not load example image"}))
189
+
190
+ return gr.update(value=json.dumps({
191
+ "status": "ok",
192
+ "query": ex["query"],
193
+ "image": img_b64,
194
+ "model": ex["model"],
195
+ "name": os.path.basename(ex["image"]),
196
+ }))
197
 
198
  def get_image_proc_params(processor) -> Dict[str, int]:
199
  ip = getattr(processor, "image_processor", None)
 
200
  default_min = 256 * 256
201
  default_max = 1280 * 1280
 
202
  patch_size = getattr(ip, "patch_size", 14)
203
  merge_size = getattr(ip, "merge_size", 2)
204
  min_pixels = getattr(ip, "min_pixels", default_min)
205
  max_pixels = getattr(ip, "max_pixels", default_max)
206
 
 
207
  size_config = getattr(ip, "size", {})
208
  if isinstance(size_config, dict):
209
  if "shortest_edge" in size_config:
 
211
  if "longest_edge" in size_config:
212
  max_pixels = size_config["longest_edge"]
213
 
214
+ if min_pixels is None:
215
+ min_pixels = default_min
216
+ if max_pixels is None:
217
+ max_pixels = default_max
218
 
219
  return {
220
  "patch_size": patch_size,
 
224
  }
225
 
226
  def apply_chat_template_compat(processor, messages: List[Dict[str, Any]], thinking: bool = True) -> str:
 
227
  if hasattr(processor, "apply_chat_template"):
228
  try:
229
  return processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, thinking=thinking)
230
  except TypeError:
 
231
  return processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
232
+
233
  tok = getattr(processor, "tokenizer", None)
234
  if tok is not None and hasattr(tok, "apply_chat_template"):
235
  return tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
236
+
237
  raise AttributeError("Could not apply chat template.")
238
 
239
  def trim_generated(generated_ids, inputs):
 
246
 
247
  def get_fara_prompt(task, image):
248
  OS_SYSTEM_PROMPT = """You are a GUI agent. You are given a task and a screenshot of the current status.
249
+ You need to generate the next action to complete the task.
250
+ Output your action inside a <tool_call> block using JSON format.
251
+ Include "coordinate": [x, y] in pixels for interactions.
252
+ Examples:
253
+ <tool_call>{"name": "User", "arguments": {"action": "click", "coordinate": [400, 300]}}</tool_call>
254
+ <tool_call>{"name": "User", "arguments": {"action": "type", "coordinate": [100, 200], "text": "hello"}}</tool_call>
255
+ """
256
  return [
257
  {"role": "system", "content": [{"type": "text", "text": OS_SYSTEM_PROMPT}]},
258
  {"role": "user", "content": [{"type": "image", "image": image}, {"type": "text", "text": f"Instruction: {task}"}]},
 
264
  "output a click position as Click(x, y) with x num pixels from the left edge "
265
  "and y num pixels from the top edge."
266
  )
267
+ return [{
268
+ "role": "user",
269
+ "content": [
270
+ {"type": "image", "image": image},
271
+ {"type": "text", "text": f"{guidelines}\n{task}"}
272
+ ],
273
+ }]
 
 
274
 
275
  def get_holo2_prompt(task, image):
276
  schema_str = '{"properties": {"x": {"description": "The x coordinate, normalized between 0 and 1000.", "ge": 0, "le": 1000, "title": "X", "type": "integer"}, "y": {"description": "The y coordinate, normalized between 0 and 1000.", "ge": 0, "le": 1000, "title": "Y", "type": "integer"}}, "required": ["x", "y"], "title": "ClickCoordinates", "type": "object"}'
 
277
  prompt = f"""Localize an element on the GUI image according to the provided target and output a click position.
278
+ * You must output a valid JSON following the format: {schema_str}
279
+ Your target is:"""
280
+ return [{
281
+ "role": "user",
282
+ "content": [
283
+ {"type": "image", "image": image},
284
+ {"type": "text", "text": f"{prompt}\n{task}"},
285
+ ],
286
+ }]
 
 
 
287
 
288
  def get_actio_prompt(task, image):
289
  system_prompt = (
 
308
  def parse_click_response(text: str) -> List[Dict]:
309
  actions = []
310
  text = text.strip()
311
+
 
 
312
  matches_click = re.findall(r"(?:click|left_click|right_click|double_click)\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)", text, re.IGNORECASE)
313
  for m in matches_click:
314
  actions.append({"type": "click", "x": int(m[0]), "y": int(m[1]), "text": "", "norm": False})
 
320
  matches_box = re.findall(r"start_box=['\"]?\(\s*(\d+)\s*,\s*(\d+)\s*\)['\"]?", text, re.IGNORECASE)
321
  for m in matches_box:
322
  actions.append({"type": "click", "x": int(m[0]), "y": int(m[1]), "text": "", "norm": False})
323
+
 
324
  if not actions:
325
  matches_tuple = re.findall(r"(?:^|\s)\(\s*(\d+)\s*,\s*(\d+)\s*\)(?:$|\s|,)", text)
326
  for m in matches_tuple:
 
344
  })
345
  except Exception as e:
346
  print(f"Error parsing Fara JSON: {e}")
 
347
  return actions
348
 
349
  def parse_holo2_response(response: str) -> List[Dict]:
350
  actions = []
351
  try:
352
  data = json.loads(response.strip())
353
+ if "x" in data and "y" in data:
354
+ actions.append({"type": "click", "x": int(data["x"]), "y": int(data["y"]), "text": "*", "norm": True})
355
  return actions
356
+ except Exception:
357
  pass
358
 
359
  match = re.search(r"\{\s*['\"]x['\"]\s*:\s*(\d+)\s*,\s*['\"]y['\"]\s*:\s*(\d+)\s*\}", response)
360
  if match:
361
  actions.append({
362
+ "type": "click",
363
+ "x": int(match.group(1)),
364
+ "y": int(match.group(2)),
365
+ "text": "Holo2",
366
+ "norm": True
367
  })
 
368
  return actions
369
 
370
  def parse_actio_response(response: str) -> List[Dict]:
 
 
371
  actions = []
 
372
  matches = re.findall(r"([a-zA-Z_]+)\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)", response)
373
  for action_name, x, y in matches:
374
  actions.append({
 
376
  "x": int(x),
377
  "y": int(y),
378
  "text": "",
379
+ "norm": False
380
  })
381
  return actions
382
 
383
+ def create_localized_image(original_image: Image.Image, actions: List[Dict]) -> Optional[Image.Image]:
384
+ if not actions:
385
+ return original_image
386
+
387
  img_copy = original_image.copy()
388
  draw = ImageDraw.Draw(img_copy)
389
+
390
  try:
391
  font = ImageFont.load_default(size=18)
392
+ except Exception:
393
  font = ImageFont.load_default()
394
+
395
  for act in actions:
396
+ x = int(act["x"])
397
+ y = int(act["y"])
398
+ color = "#ff3333" if "click" in act["type"].lower() else "#3b82f6"
399
+
 
 
 
 
400
  line_len = 15
401
  width = 4
402
+
403
+ draw.line((x - line_len, y, x + line_len, y), fill=color, width=width)
404
+ draw.line((x, y - line_len, x, y + line_len), fill=color, width=width)
405
+
 
 
406
  r = 20
407
+ draw.ellipse([x - r, y - r, x + r, y + r], outline=color, width=3)
408
+
409
  label = f"{act['type']}"
410
+ if act.get("text"):
411
+ label += f': "{act["text"]}"'
412
+
413
+ text_pos = (x + 25, y - 15)
 
414
  try:
415
  bbox = draw.textbbox(text_pos, label, font=font)
416
+ padded_bbox = (bbox[0] - 4, bbox[1] - 2, bbox[2] + 4, bbox[3] + 2)
417
  draw.rectangle(padded_bbox, fill="yellow", outline=color)
418
  draw.text(text_pos, label, fill="black", font=font)
419
+ except Exception:
420
+ draw.text(text_pos, label, fill="white", font=font)
421
 
422
  return img_copy
423
 
424
+ def calc_timeout_process(*args, **kwargs):
425
+ gpu_timeout = kwargs.get("gpu_timeout", None)
426
+ if gpu_timeout is None and args:
427
+ gpu_timeout = args[-1]
428
+ try:
429
+ return int(gpu_timeout)
430
+ except Exception:
431
+ return 60
432
+
433
+ @spaces.GPU(duration=calc_timeout_process)
434
+ def process_screenshot_stream(model_choice: str, task: str, image: Image.Image, gpu_timeout: int = 60):
435
+ try:
436
+ if image is None:
437
+ yield json.dumps({"status": "error", "text": "[ERROR] Please upload an image.", "annotated": ""})
438
+ return
439
+ if not task or not task.strip():
440
+ yield json.dumps({"status": "error", "text": "[ERROR] Please provide a task instruction.", "annotated": ""})
441
+ return
442
+ if len(str(task)) > MAX_INPUT_TEXT_LENGTH * 8:
443
+ yield json.dumps({"status": "error", "text": "[ERROR] Task instruction is too long.", "annotated": ""})
444
+ return
445
+ if model_choice not in MODEL_MAP:
446
+ yield json.dumps({"status": "error", "text": "[ERROR] Invalid model selected.", "annotated": ""})
447
+ return
448
+
449
+ input_pil_image = image.convert("RGB")
450
+ orig_w, orig_h = input_pil_image.size
451
+ raw_response = ""
452
+ actions = []
453
+
454
+ if model_choice == "Fara-7B":
455
+ if model_v is None:
456
+ yield json.dumps({"status": "error", "text": "[ERROR] Fara model failed to load.", "annotated": ""})
457
+ return
458
+
459
+ messages = get_fara_prompt(task, input_pil_image)
460
+ text_prompt = processor_v.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
461
+ image_inputs, video_inputs = process_vision_info(messages)
462
+
463
+ inputs = processor_v(
464
+ text=[text_prompt],
465
+ images=image_inputs,
466
+ videos=video_inputs,
467
+ padding=True,
468
+ return_tensors="pt"
469
+ ).to(device)
470
+
471
+ with torch.no_grad():
472
+ generated_ids = model_v.generate(**inputs, max_new_tokens=512)
473
+
474
+ generated_ids = trim_generated(generated_ids, inputs)
475
+ raw_response = processor_v.batch_decode(generated_ids, skip_special_tokens=True)[0]
476
+ actions = parse_fara_response(raw_response)
477
+
478
+ elif model_choice == "Holo2-4B":
479
+ if model_h is None:
480
+ yield json.dumps({"status": "error", "text": "[ERROR] Holo2 model failed to load.", "annotated": ""})
481
+ return
482
+
483
+ ip_params = get_image_proc_params(processor_h)
484
+ resized_h, resized_w = smart_resize(
485
+ input_pil_image.height,
486
+ input_pil_image.width,
487
+ factor=ip_params["patch_size"] * ip_params["merge_size"],
488
+ min_pixels=ip_params["min_pixels"],
489
+ max_pixels=ip_params["max_pixels"]
490
+ )
491
+ proc_image = input_pil_image.resize((resized_w, resized_h), Image.Resampling.LANCZOS)
492
+
493
+ messages = get_holo2_prompt(task, proc_image)
494
+ text_prompt = apply_chat_template_compat(processor_h, messages, thinking=False)
495
+
496
+ inputs = processor_h(text=[text_prompt], images=[proc_image], padding=True, return_tensors="pt")
497
+ inputs = {k: v.to(device) for k, v in inputs.items()}
498
+
499
+ with torch.no_grad():
500
+ generated_ids = model_h.generate(**inputs, max_new_tokens=128)
501
+
502
+ generated_ids = trim_generated(generated_ids, inputs)
503
+ raw_response = processor_h.batch_decode(generated_ids, skip_special_tokens=True)[0]
504
+ actions = parse_holo2_response(raw_response)
505
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
506
  for a in actions:
507
+ if a.get("norm", False):
508
+ a["x"] = (a["x"] / 1000.0) * orig_w
509
+ a["y"] = (a["y"] / 1000.0) * orig_h
 
 
 
 
 
 
 
 
 
 
 
 
 
510
 
511
+ elif model_choice == "UI-TARS-1.5-7B":
512
+ if model_x is None:
513
+ yield json.dumps({"status": "error", "text": "[ERROR] UI-TARS model failed to load.", "annotated": ""})
514
+ return
 
 
 
 
515
 
516
+ ip_params = get_image_proc_params(processor_x)
517
+ resized_h, resized_w = smart_resize(
518
+ input_pil_image.height,
519
+ input_pil_image.width,
520
+ factor=ip_params["patch_size"] * ip_params["merge_size"],
521
+ min_pixels=ip_params["min_pixels"],
522
+ max_pixels=ip_params["max_pixels"]
523
  )
524
+ proc_image = input_pil_image.resize((resized_w, resized_h), Image.Resampling.LANCZOS)
525
 
526
+ messages = get_localization_prompt(task, proc_image)
527
+ text_prompt = apply_chat_template_compat(processor_x, messages)
528
+
529
+ inputs = processor_x(text=[text_prompt], images=[proc_image], padding=True, return_tensors="pt")
530
+ inputs = {k: v.to(device) for k, v in inputs.items()}
531
+
532
+ with torch.no_grad():
533
+ generated_ids = model_x.generate(**inputs, max_new_tokens=128)
534
+
535
+ generated_ids = trim_generated(generated_ids, inputs)
536
+ raw_response = processor_x.batch_decode(generated_ids, skip_special_tokens=True)[0]
537
+ actions = parse_click_response(raw_response)
538
+
539
+ if resized_w > 0 and resized_h > 0:
540
+ scale_x = orig_w / resized_w
541
+ scale_y = orig_h / resized_h
542
+ for a in actions:
543
+ a["x"] = int(a["x"] * scale_x)
544
+ a["y"] = int(a["y"] * scale_y)
545
+
546
+ elif model_choice == "ActIO-UI-7B":
547
+ if model_act is None:
548
+ yield json.dumps({"status": "error", "text": "[ERROR] ActIO model failed to load.", "annotated": ""})
549
+ return
550
+
551
+ messages = get_actio_prompt(task, input_pil_image)
552
+ text_prompt = processor_act.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
553
+
554
+ inputs = processor_act(
555
+ text=[text_prompt],
556
+ images=[input_pil_image],
557
+ padding=True,
558
+ return_tensors="pt"
559
+ )
560
+ inputs = {k: v.to(device) for k, v in inputs.items()}
561
+
562
+ with torch.no_grad():
563
+ generated_ids = model_act.generate(
564
+ **inputs,
565
+ max_new_tokens=1024,
566
+ do_sample=False,
567
+ )
568
+
569
+ generated_ids = trim_generated(generated_ids, inputs)
570
+ raw_response = processor_act.batch_decode(
571
+ generated_ids,
572
+ skip_special_tokens=True,
573
+ clean_up_tokenization_spaces=False
574
+ )[0]
575
+ actions = parse_actio_response(raw_response)
576
+
577
+ annotated_image = create_localized_image(input_pil_image, actions)
578
+ annotated_b64 = pil_to_data_url(annotated_image, "JPEG") if annotated_image else pil_to_data_url(input_pil_image, "JPEG")
579
+
580
+ yield json.dumps({
581
+ "status": "done",
582
+ "text": raw_response,
583
+ "annotated": annotated_b64
584
+ })
585
+
586
+ except Exception as e:
587
+ yield json.dumps({"status": "error", "text": f"[ERROR] {str(e)}", "annotated": ""})
588
+ finally:
589
+ gc.collect()
590
+ if torch.cuda.is_available():
591
+ torch.cuda.empty_cache()
592
+
593
+ def run_cua(model_name, text, image_b64, gpu_timeout_v):
594
+ try:
595
+ image = b64_to_pil(image_b64)
596
+ yield from process_screenshot_stream(
597
+ model_choice=model_name,
598
+ task=text,
599
+ image=image,
600
+ gpu_timeout=gpu_timeout_v,
601
+ )
602
+ except Exception as e:
603
+ yield json.dumps({"status": "error", "text": f"[ERROR] {str(e)}", "annotated": ""})
604
+
605
+ def noop():
606
+ return None
607
+
608
+ CUBE_SVG = """
609
+ <svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
610
+ <path fill="white" d="M12 2 4 6v12l8 4 8-4V6l-8-4Zm0 2.2 5.6 2.8L12 9.8 6.4 7 12 4.2Zm-6 4.5 5 2.5v8.6l-5-2.5V8.7Zm7 11.1v-8.6l5-2.5v8.6l-5 2.5Z"/>
611
+ </svg>
612
+ """
613
+
614
+ UPLOAD_PREVIEW_SVG = f"""
615
+ <svg viewBox="0 0 80 80" fill="none" xmlns="http://www.w3.org/2000/svg">
616
+ <rect x="8" y="14" width="64" height="52" rx="6" fill="none" stroke="{ACCENT}" stroke-width="2" stroke-dasharray="4 3"/>
617
+ <polygon points="12,62 30,40 42,50 54,34 68,62" fill="rgba(255,255,0,0.14)" stroke="{ACCENT}" stroke-width="1.5"/>
618
+ <circle cx="28" cy="30" r="6" fill="rgba(255,255,0,0.2)" stroke="{ACCENT}" stroke-width="1.5"/>
619
+ </svg>
620
+ """
621
+
622
+ ANNOTATION_PLACEHOLDER_SVG = f"""
623
+ <svg viewBox="0 0 120 120" xmlns="http://www.w3.org/2000/svg" fill="none">
624
+ <path d="M60 16 24 34v52l36 18 36-18V34L60 16Z" stroke="{ACCENT}" stroke-width="3"/>
625
+ <path d="M24 34 60 52l36-18M60 52v52" stroke="{ACCENT}" stroke-width="2.5"/>
626
+ </svg>
627
+ """
628
+
629
+ COPY_SVG = f"""<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="{ACCENT}" d="M16 1H4C2.9 1 2 1.9 2 3v12h2V3h12V1zm3 4H8C6.9 5 6 5.9 6 7v14c0 1.1.9 2 2 2h11c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm0 16H8V7h11v14z"/></svg>"""
630
+ SAVE_SVG = f"""<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="{ACCENT}" d="M17 3H5a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2V7l-4-4zM7 5h8v4H7V5zm12 14H5v-6h14v6z"/></svg>"""
631
+
632
+ MODEL_TABS_HTML = "".join([
633
+ f'<button class="model-tab{" active" if m == "Fara-7B" else ""}" data-model="{m}"><span class="model-tab-label">{m}</span></button>'
634
+ for m in MODEL_CHOICES
635
+ ])
636
+
637
+
638
+ css = f"""
639
+ @import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500;600&display=swap');
640
+ *{{box-sizing:border-box;margin:0;padding:0}}
641
+ html,body{{height:100%;overflow-x:hidden}}
642
+ body,.gradio-container{{
643
+ background:#0f0f13!important;
644
+ font-family:'Inter',system-ui,-apple-system,sans-serif!important;
645
+ font-size:14px!important;color:#e4e4e7!important;min-height:100vh;overflow-x:hidden;
646
+ }}
647
+ .dark body,.dark .gradio-container{{background:#0f0f13!important;color:#e4e4e7!important}}
648
+ footer{{display:none!important}}
649
+ .hidden-input{{display:none!important;height:0!important;overflow:hidden!important;margin:0!important;padding:0!important}}
650
+ #gradio-run-btn,#example-load-btn{{
651
+ position:absolute!important;left:-9999px!important;top:-9999px!important;
652
+ width:1px!important;height:1px!important;opacity:0.01!important;
653
+ pointer-events:none!important;overflow:hidden!important;
654
+ }}
655
+
656
+ .app-shell{{
657
+ background:#18181b;border:1px solid #27272a;border-radius:16px;
658
+ margin:12px auto;max-width:1440px;overflow:hidden;
659
+ box-shadow:0 25px 50px -12px rgba(0,0,0,.6),0 0 0 1px rgba(255,255,255,.03);
660
+ }}
661
+ .app-header{{
662
+ background:linear-gradient(135deg,#18181b,#1e1e24);border-bottom:1px solid #27272a;
663
+ padding:14px 24px;display:flex;align-items:center;justify-content:space-between;flex-wrap:wrap;gap:12px;
664
+ }}
665
+ .app-header-left{{display:flex;align-items:center;gap:12px}}
666
+ .app-logo{{
667
+ width:38px;height:38px;background:linear-gradient(135deg,{ACCENT},#fff06a,#fff7b2);
668
+ border-radius:10px;display:flex;align-items:center;justify-content:center;
669
+ box-shadow:0 4px 12px rgba(255,255,0,.30);
670
+ }}
671
+ .app-logo svg{{width:22px;height:22px;fill:#111;flex-shrink:0}}
672
+ .app-title{{
673
+ font-size:18px;font-weight:700;background:linear-gradient(135deg,#f5f5f5,#d9d9a7);
674
+ -webkit-background-clip:text;-webkit-text-fill-color:transparent;letter-spacing:-.3px;
675
+ }}
676
+ .app-badge{{
677
+ font-size:11px;font-weight:600;padding:3px 10px;border-radius:20px;
678
+ background:rgba(255,255,0,.10);color:#fff8a6;border:1px solid rgba(255,255,0,.24);letter-spacing:.3px;
679
+ }}
680
+ .app-badge.fast{{background:rgba(255,255,0,.08);color:#fff39a;border:1px solid rgba(255,255,0,.20)}}
681
+
682
+ .model-tabs-bar{{
683
+ background:#18181b;border-bottom:1px solid #27272a;padding:10px 16px;
684
+ display:flex;gap:8px;align-items:center;flex-wrap:wrap;
685
+ }}
686
+ .model-tab{{
687
+ display:inline-flex;align-items:center;justify-content:center;gap:6px;
688
+ min-width:32px;height:34px;background:transparent;border:1px solid #27272a;
689
+ border-radius:999px;cursor:pointer;font-size:12px;font-weight:600;padding:0 12px;
690
+ color:#ffffff!important;transition:all .15s ease;
691
+ }}
692
+ .model-tab:hover{{background:rgba(255,255,0,.10);border-color:rgba(255,255,0,.35)}}
693
+ .model-tab.active{{background:rgba(255,255,0,.16);border-color:{ACCENT};color:#fff!important;box-shadow:0 0 0 2px rgba(255,255,0,.08)}}
694
+ .model-tab-label{{font-size:12px;color:#ffffff!important;font-weight:600}}
695
+
696
+ .app-main-row{{display:flex;gap:0;flex:1;overflow:hidden}}
697
+ .app-main-left{{flex:1;display:flex;flex-direction:column;min-width:0;border-right:1px solid #27272a}}
698
+ .app-main-right{{width:520px;display:flex;flex-direction:column;flex-shrink:0;background:#18181b}}
699
+
700
+ #image-drop-zone{{
701
+ position:relative;background:#09090b;height:460px;min-height:460px;max-height:460px;
702
+ overflow:hidden;
703
+ }}
704
+ #image-drop-zone.drag-over{{outline:2px solid {ACCENT};outline-offset:-2px;background:rgba(255,255,0,.04)}}
705
+ .upload-prompt-modern{{
706
+ position:absolute;inset:0;display:flex;align-items:center;justify-content:center;
707
+ padding:20px;z-index:20;overflow:hidden;
708
+ }}
709
+ .upload-click-area{{
710
+ display:flex;flex-direction:column;align-items:center;justify-content:center;
711
+ cursor:pointer;padding:28px 36px;max-width:92%;max-height:92%;
712
+ border:2px dashed #3f3f46;border-radius:16px;
713
+ background:rgba(255,255,0,.03);transition:all .2s ease;gap:8px;text-align:center;
714
+ overflow:hidden;
715
+ }}
716
+ .upload-click-area:hover{{background:rgba(255,255,0,.08);border-color:{ACCENT};transform:scale(1.02)}}
717
+ .upload-click-area:active{{background:rgba(255,255,0,.12);transform:scale(.99)}}
718
+ .upload-click-area svg{{width:86px;height:86px;max-width:100%;flex-shrink:0}}
719
+ .upload-main-text{{color:#a1a1aa;font-size:14px;font-weight:600;margin-top:4px}}
720
+ .upload-sub-text{{color:#71717a;font-size:12px}}
721
+
722
+ .single-preview-wrap{{
723
+ width:100%;height:100%;display:none;align-items:center;justify-content:center;padding:16px;
724
+ overflow:hidden;
725
+ }}
726
+ .single-preview-card{{
727
+ width:100%;height:100%;max-width:100%;max-height:100%;border-radius:14px;
728
+ overflow:hidden;border:1px solid #27272a;background:#111114;
729
+ display:flex;align-items:center;justify-content:center;position:relative;
730
+ }}
731
+ .single-preview-card img{{
732
+ width:100%;height:100%;max-width:100%;max-height:100%;
733
+ object-fit:contain;display:block;
734
+ }}
735
+ .preview-overlay-actions{{
736
+ position:absolute;top:12px;right:12px;display:flex;gap:8px;z-index:5;
737
+ }}
738
+ .preview-action-btn{{
739
+ display:inline-flex;align-items:center;justify-content:center;
740
+ min-width:34px;height:34px;padding:0 12px;background:rgba(0,0,0,.65);
741
+ border:1px solid rgba(255,255,255,.14);border-radius:10px;cursor:pointer;
742
+ color:#fff!important;font-size:12px;font-weight:600;transition:all .15s ease;
743
+ }}
744
+ .preview-action-btn:hover{{background:{ACCENT};border-color:{ACCENT};color:#121200!important}}
745
+
746
+ .hint-bar{{
747
+ background:rgba(255,255,0,.05);border-top:1px solid #27272a;border-bottom:1px solid #27272a;
748
+ padding:10px 20px;font-size:13px;color:#a1a1aa;line-height:1.7;
749
+ }}
750
+ .hint-bar b{{color:#fff6a0;font-weight:600}}
751
+ .hint-bar kbd{{
752
+ display:inline-block;padding:1px 6px;background:#27272a;border:1px solid #3f3f46;
753
+ border-radius:4px;font-family:'JetBrains Mono',monospace;font-size:11px;color:#a1a1aa;
754
+ }}
755
+
756
+ .examples-section{{border-top:1px solid #27272a;padding:12px 16px}}
757
+ .examples-title{{
758
+ font-size:12px;font-weight:600;color:#71717a;text-transform:uppercase;
759
+ letter-spacing:.8px;margin-bottom:10px;
760
+ }}
761
+ .examples-scroll{{display:flex;gap:10px;overflow-x:auto;padding-bottom:8px}}
762
+ .examples-scroll::-webkit-scrollbar{{height:6px}}
763
+ .examples-scroll::-webkit-scrollbar-track{{background:#09090b;border-radius:3px}}
764
+ .examples-scroll::-webkit-scrollbar-thumb{{background:#27272a;border-radius:3px}}
765
+ .examples-scroll::-webkit-scrollbar-thumb:hover{{background:#3f3f46}}
766
+ .example-card{{
767
+ flex-shrink:0;width:220px;background:#09090b;border:1px solid #27272a;
768
+ border-radius:10px;overflow:hidden;cursor:pointer;transition:all .2s ease;
769
+ }}
770
+ .example-card:hover{{border-color:{ACCENT};transform:translateY(-2px);box-shadow:0 4px 12px rgba(255,255,0,.14)}}
771
+ .example-card.loading{{opacity:.5;pointer-events:none}}
772
+ .example-thumb-wrap{{height:120px;overflow:hidden;background:#18181b}}
773
+ .example-thumb-wrap img{{width:100%;height:100%;object-fit:cover}}
774
+ .example-thumb-placeholder{{
775
+ width:100%;height:100%;display:flex;align-items:center;justify-content:center;
776
+ background:#18181b;color:#3f3f46;font-size:11px;
777
+ }}
778
+ .example-meta-row{{padding:6px 10px;display:flex;align-items:center;gap:6px}}
779
+ .example-badge{{
780
+ display:inline-flex;padding:2px 7px;background:rgba(255,255,0,.12);border-radius:4px;
781
+ font-size:10px;font-weight:600;color:#fff6a0;font-family:'JetBrains Mono',monospace;white-space:nowrap;
782
+ }}
783
+ .example-prompt-text{{
784
+ padding:0 10px 8px;font-size:11px;color:#a1a1aa;line-height:1.4;
785
+ display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden;
786
+ }}
787
+
788
+ .panel-card{{border-bottom:1px solid #27272a}}
789
+ .panel-card-title{{
790
+ padding:12px 20px;font-size:12px;font-weight:600;color:#71717a;
791
+ text-transform:uppercase;letter-spacing:.8px;border-bottom:1px solid rgba(39,39,42,.6);
792
+ }}
793
+ .panel-card-body{{padding:16px 20px;display:flex;flex-direction:column;gap:8px}}
794
+ .modern-label{{font-size:13px;font-weight:500;color:#a1a1aa;margin-bottom:4px;display:block}}
795
+ .modern-textarea{{
796
+ width:100%;background:#09090b;border:1px solid #27272a;border-radius:8px;
797
+ padding:10px 14px;font-family:'Inter',sans-serif;font-size:14px;color:#e4e4e7;
798
+ resize:none;outline:none;min-height:100px;transition:border-color .2s;
799
+ }}
800
+ .modern-textarea:focus{{border-color:{ACCENT};box-shadow:0 0 0 3px rgba(255,255,0,.14)}}
801
+ .modern-textarea::placeholder{{color:#3f3f46}}
802
+ .modern-textarea.error-flash{{
803
+ border-color:#ef4444!important;box-shadow:0 0 0 3px rgba(239,68,68,.2)!important;animation:shake .4s ease;
804
+ }}
805
+ @keyframes shake{{0%,100%{{transform:translateX(0)}}20%,60%{{transform:translateX(-4px)}}40%,80%{{transform:translateX(4px)}}}}
806
+
807
+ .toast-notification{{
808
+ position:fixed;top:24px;left:50%;transform:translateX(-50%) translateY(-120%);
809
+ z-index:9999;padding:10px 24px;border-radius:10px;font-family:'Inter',sans-serif;
810
+ font-size:14px;font-weight:600;display:flex;align-items:center;gap:8px;
811
+ box-shadow:0 8px 24px rgba(0,0,0,.5);
812
+ transition:transform .35s cubic-bezier(.34,1.56,.64,1),opacity .35s ease;opacity:0;pointer-events:none;
813
+ }}
814
+ .toast-notification.visible{{transform:translateX(-50%) translateY(0);opacity:1;pointer-events:auto}}
815
+ .toast-notification.error{{background:linear-gradient(135deg,#dc2626,#b91c1c);color:#fff;border:1px solid rgba(255,255,255,.15)}}
816
+ .toast-notification.warning{{background:linear-gradient(135deg,#b7b700,#8f8f00);color:#fff;border:1px solid rgba(255,255,255,.15)}}
817
+ .toast-notification.info{{background:linear-gradient(135deg,#d4d400,{ACCENT});color:#111;border:1px solid rgba(255,255,255,.15)}}
818
+ .toast-notification .toast-icon{{font-size:16px;line-height:1}}
819
+ .toast-notification .toast-text{{line-height:1.3}}
820
 
821
+ .btn-run{{
822
+ display:flex;align-items:center;justify-content:center;gap:8px;width:100%;
823
+ background:linear-gradient(135deg,{ACCENT},#d8d800);border:none;border-radius:10px;
824
+ padding:12px 24px;cursor:pointer;font-size:15px;font-weight:700;font-family:'Inter',sans-serif;
825
+ color:#141400!important;-webkit-text-fill-color:#141400!important;
826
+ transition:all .2s ease;letter-spacing:-.2px;
827
+ box-shadow:0 4px 16px rgba(255,255,0,.25),inset 0 1px 0 rgba(255,255,255,.18);
828
+ }}
829
+ .btn-run:hover{{
830
+ background:linear-gradient(135deg,#ffff7a,{ACCENT});transform:translateY(-1px);
831
+ box-shadow:0 6px 24px rgba(255,255,0,.35),inset 0 1px 0 rgba(255,255,255,.22);
832
+ }}
833
+ .btn-run:active{{transform:translateY(0);box-shadow:0 2px 8px rgba(255,255,0,.25)}}
834
 
835
+ .annot-frame{{border-bottom:1px solid #27272a;display:flex;flex-direction:column;position:relative}}
836
+ .annot-title{{
837
+ padding:10px 20px;font-size:13px;font-weight:700;text-transform:uppercase;
838
+ letter-spacing:.8px;border-bottom:1px solid rgba(39,39,42,.6);color:#fff
839
+ }}
840
+ .annot-body{{
841
+ background:#09090b;height:340px;display:flex;align-items:center;justify-content:center;
842
+ padding:12px;position:relative;overflow:hidden;
843
+ }}
844
+ .annot-body img{{
845
+ max-width:100%;max-height:100%;object-fit:contain;border:1px solid #27272a;
846
+ border-radius:10px;background:#111114;display:none;position:relative;z-index:2;
847
+ }}
848
+ .annot-placeholder{{
849
+ position:absolute;inset:0;display:flex;flex-direction:column;align-items:center;justify-content:center;
850
+ gap:10px;color:#666;z-index:1;padding:16px;text-align:center;
851
+ }}
852
+ .annot-placeholder svg{{width:92px;height:92px;max-width:100%;opacity:.95}}
853
+ .annot-placeholder-title{{font-size:13px;font-weight:600;color:#fff6a0}}
854
+ .annot-placeholder-sub{{font-size:12px;color:#666;max-width:260px;line-height:1.5}}
855
 
856
+ .output-frame{{border-bottom:1px solid #27272a;display:flex;flex-direction:column;position:relative}}
857
+ .output-frame .out-title,
858
+ .output-frame .out-title *,
859
+ #output-title-label{{
860
+ color:#ffffff!important;
861
+ -webkit-text-fill-color:#ffffff!important;
862
+ }}
863
+ .output-frame .out-title{{
864
+ padding:10px 20px;font-size:13px;font-weight:700;
865
+ text-transform:uppercase;letter-spacing:.8px;border-bottom:1px solid rgba(39,39,42,.6);
866
+ display:flex;align-items:center;justify-content:space-between;gap:8px;flex-wrap:wrap;
867
+ }}
868
+ .out-title-right{{display:flex;gap:8px;align-items:center}}
869
+ .out-action-btn{{
870
+ display:inline-flex;align-items:center;justify-content:center;background:rgba(255,255,0,.10);
871
+ border:1px solid rgba(255,255,0,.2);border-radius:6px;cursor:pointer;padding:3px 10px;
872
+ font-size:11px;font-weight:500;color:#fff6a0!important;gap:4px;height:24px;transition:all .15s;
873
+ }}
874
+ .out-action-btn:hover{{background:rgba(255,255,0,.2);border-color:rgba(255,255,0,.35);color:#ffffff!important}}
875
+ .out-action-btn svg{{width:12px;height:12px;fill:{ACCENT}}}
876
+ .output-frame .out-body{{
877
+ flex:1;background:#09090b;display:flex;align-items:stretch;justify-content:stretch;
878
+ overflow:hidden;min-height:300px;position:relative;
879
+ }}
880
+ .output-scroll-wrap{{width:100%;height:100%;padding:0;overflow:hidden}}
881
+ .output-textarea{{
882
+ width:100%;height:300px;min-height:300px;max-height:300px;background:#09090b;color:#e4e4e7;
883
+ border:none;outline:none;padding:16px 18px;font-size:13px;line-height:1.6;
884
+ font-family:'JetBrains Mono',monospace;overflow:auto;resize:none;white-space:pre-wrap;
885
+ }}
886
+ .output-textarea::placeholder{{color:#52525b}}
887
+ .output-textarea.error-flash{{box-shadow:inset 0 0 0 2px rgba(239,68,68,.6)}}
888
 
889
+ .modern-loader{{
890
+ display:none;position:absolute;top:0;left:0;right:0;bottom:0;background:rgba(9,9,11,.92);
891
+ z-index:15;flex-direction:column;align-items:center;justify-content:center;gap:16px;backdrop-filter:blur(4px);
892
+ }}
893
+ .modern-loader.active{{display:flex}}
894
+ .modern-loader .loader-spinner{{
895
+ width:36px;height:36px;border:3px solid #27272a;border-top-color:{ACCENT};
896
+ border-radius:50%;animation:spin .8s linear infinite;
897
+ }}
898
+ @keyframes spin{{to{{transform:rotate(360deg)}}}}
899
+ .modern-loader .loader-text{{font-size:13px;color:#a1a1aa;font-weight:500}}
900
+ .loader-bar-track{{width:200px;height:4px;background:#27272a;border-radius:2px;overflow:hidden}}
901
+ .loader-bar-fill{{
902
+ height:100%;background:linear-gradient(90deg,{ACCENT},#ffff94,{ACCENT});
903
+ background-size:200% 100%;animation:shimmer 1.5s ease-in-out infinite;border-radius:2px;
904
+ }}
905
+ @keyframes shimmer{{0%{{background-position:200% 0}}100%{{background-position:-200% 0}}}}
906
+
907
+ .settings-group{{border:1px solid #27272a;border-radius:10px;margin:12px 16px;padding:0;overflow:hidden}}
908
+ .settings-group-title{{
909
+ font-size:12px;font-weight:600;color:#71717a;text-transform:uppercase;letter-spacing:.8px;
910
+ padding:10px 16px;border-bottom:1px solid #27272a;background:rgba(24,24,27,.5);
911
+ }}
912
+ .settings-group-body{{padding:14px 16px;display:flex;flex-direction:column;gap:12px}}
913
+ .slider-row{{display:flex;align-items:center;gap:10px;min-height:28px}}
914
+ .slider-row label{{font-size:13px;font-weight:500;color:#a1a1aa;min-width:118px;flex-shrink:0}}
915
+ .slider-row input[type="range"]{{
916
+ flex:1;-webkit-appearance:none;appearance:none;height:6px;background:#27272a;
917
+ border-radius:3px;outline:none;min-width:0;
918
+ }}
919
+ .slider-row input[type="range"]::-webkit-slider-thumb{{
920
+ -webkit-appearance:none;width:16px;height:16px;background:linear-gradient(135deg,{ACCENT},#d8d800);
921
+ border-radius:50%;cursor:pointer;box-shadow:0 2px 6px rgba(255,255,0,.35);transition:transform .15s;
922
+ }}
923
+ .slider-row input[type="range"]::-webkit-slider-thumb:hover{{transform:scale(1.2)}}
924
+ .slider-row input[type="range"]::-moz-range-thumb{{
925
+ width:16px;height:16px;background:linear-gradient(135deg,{ACCENT},#d8d800);
926
+ border-radius:50%;cursor:pointer;border:none;box-shadow:0 2px 6px rgba(255,255,0,.35);
927
+ }}
928
+ .slider-row .slider-val{{
929
+ min-width:58px;text-align:right;font-family:'JetBrains Mono',monospace;font-size:12px;
930
+ font-weight:500;padding:3px 8px;background:#09090b;border:1px solid #27272a;
931
+ border-radius:6px;color:#a1a1aa;flex-shrink:0;
932
+ }}
933
+
934
+ .app-statusbar{{
935
+ background:#18181b;border-top:1px solid #27272a;padding:6px 20px;
936
+ display:flex;gap:12px;height:34px;align-items:center;font-size:12px;
937
+ }}
938
+ .app-statusbar .sb-section{{
939
+ padding:0 12px;flex:1;display:flex;align-items:center;font-family:'JetBrains Mono',monospace;
940
+ font-size:12px;color:#52525b;overflow:hidden;white-space:nowrap;
941
+ }}
942
+ .app-statusbar .sb-section.sb-fixed{{
943
+ flex:0 0 auto;min-width:110px;text-align:center;justify-content:center;
944
+ padding:3px 12px;background:rgba(255,255,0,.08);border-radius:6px;color:#fff6a0;font-weight:500;
945
+ }}
946
+
947
+ .exp-note{{padding:10px 20px;font-size:12px;color:#52525b;border-top:1px solid #27272a;text-align:center}}
948
+ .exp-note a{{color:#fff6a0;text-decoration:none}}
949
+ .exp-note a:hover{{text-decoration:underline}}
950
+
951
+ ::-webkit-scrollbar{{width:8px;height:8px}}
952
+ ::-webkit-scrollbar-track{{background:#09090b}}
953
+ ::-webkit-scrollbar-thumb{{background:#27272a;border-radius:4px}}
954
+ ::-webkit-scrollbar-thumb:hover{{background:#3f3f46}}
955
+
956
+ @media(max-width:980px){{
957
+ .app-main-row{{flex-direction:column}}
958
+ .app-main-right{{width:100%}}
959
+ .app-main-left{{border-right:none;border-bottom:1px solid #27272a}}
960
+ }}
961
+ """
962
+
963
+ gallery_js = r"""
964
+ () => {
965
+ function init() {
966
+ if (window.__cuaInitDone) return;
967
+
968
+ const dropZone = document.getElementById('image-drop-zone');
969
+ const uploadPrompt = document.getElementById('upload-prompt');
970
+ const uploadClick = document.getElementById('upload-click-area');
971
+ const fileInput = document.getElementById('custom-file-input');
972
+ const previewWrap = document.getElementById('single-preview-wrap');
973
+ const previewImg = document.getElementById('single-preview-img');
974
+ const btnUpload = document.getElementById('preview-upload-btn');
975
+ const btnClear = document.getElementById('preview-clear-btn');
976
+ const promptInput = document.getElementById('custom-query-input');
977
+ const runBtnEl = document.getElementById('custom-run-btn');
978
+ const outputArea = document.getElementById('custom-output-textarea');
979
+ const annotImg = document.getElementById('annotated-output-img');
980
+ const annotPlaceholder = document.getElementById('annotated-output-placeholder');
981
+ const imgStatus = document.getElementById('sb-image-status');
982
+
983
+ if (!dropZone || !fileInput || !promptInput || !previewWrap || !previewImg) {
984
+ setTimeout(init, 250);
985
+ return;
986
+ }
987
+
988
+ window.__cuaInitDone = true;
989
+ let imageState = null;
990
+ let toastTimer = null;
991
+ let examplePoller = null;
992
+ let lastSeenExamplePayload = null;
993
+
994
+ function showToast(message, type) {
995
+ let toast = document.getElementById('app-toast');
996
+ if (!toast) {
997
+ toast = document.createElement('div');
998
+ toast.id = 'app-toast';
999
+ toast.className = 'toast-notification';
1000
+ toast.innerHTML = '<span class="toast-icon"></span><span class="toast-text"></span>';
1001
+ document.body.appendChild(toast);
1002
+ }
1003
+ const icon = toast.querySelector('.toast-icon');
1004
+ const text = toast.querySelector('.toast-text');
1005
+ toast.className = 'toast-notification ' + (type || 'error');
1006
+ if (type === 'warning') icon.textContent = '\u26A0';
1007
+ else if (type === 'info') icon.textContent = '\u2139';
1008
+ else icon.textContent = '\u2717';
1009
+ text.textContent = message;
1010
+ if (toastTimer) clearTimeout(toastTimer);
1011
+ void toast.offsetWidth;
1012
+ toast.classList.add('visible');
1013
+ toastTimer = setTimeout(() => toast.classList.remove('visible'), 3500);
1014
+ }
1015
+
1016
+ function showLoader() {
1017
+ const l = document.getElementById('output-loader');
1018
+ if (l) l.classList.add('active');
1019
+ const sb = document.getElementById('sb-run-state');
1020
+ if (sb) sb.textContent = 'Processing...';
1021
+ }
1022
+ function hideLoader() {
1023
+ const l = document.getElementById('output-loader');
1024
+ if (l) l.classList.remove('active');
1025
+ const sb = document.getElementById('sb-run-state');
1026
+ if (sb) sb.textContent = 'Done';
1027
+ }
1028
+ function setRunErrorState() {
1029
+ const l = document.getElementById('output-loader');
1030
+ if (l) l.classList.remove('active');
1031
+ const sb = document.getElementById('sb-run-state');
1032
+ if (sb) sb.textContent = 'Error';
1033
+ }
1034
+
1035
+ function flashPromptError() {
1036
+ promptInput.classList.add('error-flash');
1037
+ promptInput.focus();
1038
+ setTimeout(() => promptInput.classList.remove('error-flash'), 800);
1039
+ }
1040
+
1041
+ function flashOutputError() {
1042
+ if (!outputArea) return;
1043
+ outputArea.classList.add('error-flash');
1044
+ setTimeout(() => outputArea.classList.remove('error-flash'), 800);
1045
+ }
1046
+
1047
+ function getValueFromContainer(containerId) {
1048
+ const container = document.getElementById(containerId);
1049
+ if (!container) return '';
1050
+ const el = container.querySelector('textarea, input');
1051
+ return el ? (el.value || '') : '';
1052
+ }
1053
+
1054
+ function setGradioValue(containerId, value) {
1055
+ const container = document.getElementById(containerId);
1056
+ if (!container) return false;
1057
+ const el = container.querySelector('textarea, input');
1058
+ if (!el) return false;
1059
+ const proto = el.tagName === 'TEXTAREA' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
1060
+ const ns = Object.getOwnPropertyDescriptor(proto, 'value');
1061
+ if (ns && ns.set) {
1062
+ ns.set.call(el, value);
1063
+ el.dispatchEvent(new Event('input', {bubbles:true, composed:true}));
1064
+ el.dispatchEvent(new Event('change', {bubbles:true, composed:true}));
1065
+ return true;
1066
+ }
1067
+ return false;
1068
+ }
1069
+
1070
+ function syncImageToGradio() {
1071
+ setGradioValue('hidden-image-b64', imageState ? imageState.b64 : '');
1072
+ if (imgStatus) imgStatus.textContent = imageState ? '1 image uploaded' : 'No image uploaded';
1073
+ }
1074
+
1075
+ function syncPromptToGradio() {
1076
+ setGradioValue('prompt-gradio-input', promptInput.value);
1077
+ }
1078
+
1079
+ function syncModelToGradio(name) {
1080
+ setGradioValue('hidden-model-name', name);
1081
+ }
1082
+
1083
+ function updateAnnotationState(src) {
1084
+ if (!annotImg || !annotPlaceholder) return;
1085
+ if (src) {
1086
+ annotImg.src = src;
1087
+ annotImg.style.display = 'block';
1088
+ annotPlaceholder.style.display = 'none';
1089
+ } else {
1090
+ annotImg.src = '';
1091
+ annotImg.style.display = 'none';
1092
+ annotPlaceholder.style.display = 'flex';
1093
+ }
1094
+ }
1095
+
1096
+ function setPreview(b64, name) {
1097
+ imageState = {b64, name: name || 'image'};
1098
+ previewImg.src = b64;
1099
+ previewWrap.style.display = 'flex';
1100
+ if (uploadPrompt) uploadPrompt.style.display = 'none';
1101
+ syncImageToGradio();
1102
+ }
1103
+
1104
+ function clearPreview() {
1105
+ imageState = null;
1106
+ previewImg.src = '';
1107
+ previewWrap.style.display = 'none';
1108
+ if (uploadPrompt) uploadPrompt.style.display = 'flex';
1109
+ syncImageToGradio();
1110
+ updateAnnotationState('');
1111
+ }
1112
+
1113
+ window.__setPreview = setPreview;
1114
+ window.__clearPreview = clearPreview;
1115
+ window.__updateAnnotationState = updateAnnotationState;
1116
+ window.__showToast = showToast;
1117
+ window.__showLoader = showLoader;
1118
+ window.__hideLoader = hideLoader;
1119
+ window.__setRunErrorState = setRunErrorState;
1120
+
1121
+ function processFile(file) {
1122
+ if (!file) return;
1123
+ if (!file.type.startsWith('image/')) {
1124
+ showToast('Only image files are supported', 'error');
1125
+ return;
1126
+ }
1127
+ const reader = new FileReader();
1128
+ reader.onload = (e) => setPreview(e.target.result, file.name);
1129
+ reader.readAsDataURL(file);
1130
+ }
1131
+
1132
+ fileInput.addEventListener('change', (e) => {
1133
+ const file = e.target.files && e.target.files[0] ? e.target.files[0] : null;
1134
+ if (file) processFile(file);
1135
+ e.target.value = '';
1136
+ });
1137
+
1138
+ if (uploadClick) uploadClick.addEventListener('click', () => fileInput.click());
1139
+ if (btnUpload) btnUpload.addEventListener('click', () => fileInput.click());
1140
+ if (btnClear) btnClear.addEventListener('click', clearPreview);
1141
+
1142
+ dropZone.addEventListener('dragover', (e) => {
1143
+ e.preventDefault();
1144
+ dropZone.classList.add('drag-over');
1145
+ });
1146
+ dropZone.addEventListener('dragleave', (e) => {
1147
+ e.preventDefault();
1148
+ dropZone.classList.remove('drag-over');
1149
+ });
1150
+ dropZone.addEventListener('drop', (e) => {
1151
+ e.preventDefault();
1152
+ dropZone.classList.remove('drag-over');
1153
+ if (e.dataTransfer.files && e.dataTransfer.files.length) processFile(e.dataTransfer.files[0]);
1154
+ });
1155
+
1156
+ promptInput.addEventListener('input', syncPromptToGradio);
1157
+
1158
+ function activateModelTab(name) {
1159
+ document.querySelectorAll('.model-tab[data-model]').forEach(btn => {
1160
+ btn.classList.toggle('active', btn.getAttribute('data-model') === name);
1161
+ });
1162
+ syncModelToGradio(name);
1163
+ }
1164
+ window.__activateModelTab = activateModelTab;
1165
+
1166
+ document.querySelectorAll('.model-tab[data-model]').forEach(btn => {
1167
+ btn.addEventListener('click', () => activateModelTab(btn.getAttribute('data-model')));
1168
+ });
1169
+
1170
+ activateModelTab('Fara-7B');
1171
+ updateAnnotationState('');
1172
+
1173
+ function syncSlider(customId, gradioId) {
1174
+ const slider = document.getElementById(customId);
1175
+ const valSpan = document.getElementById(customId + '-val');
1176
+ if (!slider) return;
1177
+ slider.addEventListener('input', () => {
1178
+ if (valSpan) valSpan.textContent = slider.value;
1179
+ const container = document.getElementById(gradioId);
1180
+ if (!container) return;
1181
+ container.querySelectorAll('input[type="range"],input[type="number"]').forEach(el => {
1182
+ const ns = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value');
1183
+ if (ns && ns.set) {
1184
+ ns.set.call(el, slider.value);
1185
+ el.dispatchEvent(new Event('input', {bubbles:true, composed:true}));
1186
+ el.dispatchEvent(new Event('change', {bubbles:true, composed:true}));
1187
+ }
1188
+ });
1189
+ });
1190
+ }
1191
+
1192
+ syncSlider('custom-gpu-duration', 'gradio-gpu-duration');
1193
+
1194
+ function validateBeforeRun() {
1195
+ const promptVal = promptInput.value.trim();
1196
+ if (!imageState && !promptVal) {
1197
+ showToast('Please upload an image and enter your task instruction', 'error');
1198
+ flashPromptError();
1199
+ return false;
1200
+ }
1201
+ if (!imageState) {
1202
+ showToast('Please upload an image', 'error');
1203
+ return false;
1204
+ }
1205
+ if (!promptVal) {
1206
+ showToast('Please enter your task instruction', 'warning');
1207
+ flashPromptError();
1208
+ return false;
1209
+ }
1210
+ const currentModel = (document.querySelector('.model-tab.active') || {}).dataset?.model;
1211
+ if (!currentModel) {
1212
+ showToast('Please select a model', 'error');
1213
+ return false;
1214
+ }
1215
+ return true;
1216
+ }
1217
+
1218
+ window.__clickGradioRunBtn = function() {
1219
+ if (!validateBeforeRun()) return;
1220
+ syncPromptToGradio();
1221
+ syncImageToGradio();
1222
+ const active = document.querySelector('.model-tab.active');
1223
+ if (active) syncModelToGradio(active.getAttribute('data-model'));
1224
+ if (outputArea) outputArea.value = '';
1225
+ updateAnnotationState('');
1226
+ showLoader();
1227
+ setTimeout(() => {
1228
+ const gradioBtn = document.getElementById('gradio-run-btn');
1229
+ if (!gradioBtn) {
1230
+ setRunErrorState();
1231
+ if (outputArea) outputArea.value = '[ERROR] Run button not found.';
1232
+ showToast('Run button not found', 'error');
1233
+ return;
1234
+ }
1235
+ const btn = gradioBtn.querySelector('button');
1236
+ if (btn) btn.click(); else gradioBtn.click();
1237
+ }, 180);
1238
+ };
1239
+
1240
+ if (runBtnEl) runBtnEl.addEventListener('click', () => window.__clickGradioRunBtn());
1241
+
1242
+ const copyBtn = document.getElementById('copy-output-btn');
1243
+ if (copyBtn) {
1244
+ copyBtn.addEventListener('click', async () => {
1245
+ try {
1246
+ const text = outputArea ? outputArea.value : '';
1247
+ if (!text.trim()) {
1248
+ showToast('No output to copy', 'warning');
1249
+ flashOutputError();
1250
+ return;
1251
+ }
1252
+ await navigator.clipboard.writeText(text);
1253
+ showToast('Output copied to clipboard', 'info');
1254
+ } catch(e) {
1255
+ showToast('Copy failed', 'error');
1256
+ }
1257
+ });
1258
+ }
1259
+
1260
+ const saveBtn = document.getElementById('save-output-btn');
1261
+ if (saveBtn) {
1262
+ saveBtn.addEventListener('click', () => {
1263
+ const text = outputArea ? outputArea.value : '';
1264
+ if (!text.trim()) {
1265
+ showToast('No output to save', 'warning');
1266
+ flashOutputError();
1267
+ return;
1268
+ }
1269
+ const blob = new Blob([text], {type: 'text/plain;charset=utf-8'});
1270
+ const a = document.createElement('a');
1271
+ a.href = URL.createObjectURL(blob);
1272
+ a.download = 'cua_gui_operator_output.txt';
1273
+ document.body.appendChild(a);
1274
+ a.click();
1275
+ setTimeout(() => {
1276
+ URL.revokeObjectURL(a.href);
1277
+ document.body.removeChild(a);
1278
+ }, 200);
1279
+ showToast('Output saved', 'info');
1280
+ });
1281
+ }
1282
+
1283
+ function applyExamplePayload(raw) {
1284
+ try {
1285
+ const data = JSON.parse(raw);
1286
+ if (data.status === 'ok') {
1287
+ if (data.image) setPreview(data.image, data.name || 'example.png');
1288
+ if (data.query) {
1289
+ promptInput.value = data.query;
1290
+ syncPromptToGradio();
1291
+ }
1292
+ if (data.model) activateModelTab(data.model);
1293
+ document.querySelectorAll('.example-card.loading').forEach(c => c.classList.remove('loading'));
1294
+ showToast('Example loaded', 'info');
1295
+ } else if (data.status === 'error') {
1296
+ document.querySelectorAll('.example-card.loading').forEach(c => c.classList.remove('loading'));
1297
+ showToast(data.message || 'Failed to load example', 'error');
1298
+ }
1299
+ } catch (e) {
1300
+ document.querySelectorAll('.example-card.loading').forEach(c => c.classList.remove('loading'));
1301
+ }
1302
+ }
1303
+
1304
+ function startExamplePolling() {
1305
+ if (examplePoller) clearInterval(examplePoller);
1306
+ let attempts = 0;
1307
+ examplePoller = setInterval(() => {
1308
+ attempts += 1;
1309
+ const current = getValueFromContainer('example-result-data');
1310
+ if (current && current !== lastSeenExamplePayload) {
1311
+ lastSeenExamplePayload = current;
1312
+ clearInterval(examplePoller);
1313
+ examplePoller = null;
1314
+ applyExamplePayload(current);
1315
+ return;
1316
+ }
1317
+ if (attempts >= 100) {
1318
+ clearInterval(examplePoller);
1319
+ examplePoller = null;
1320
+ document.querySelectorAll('.example-card.loading').forEach(c => c.classList.remove('loading'));
1321
+ showToast('Example load timed out', 'error');
1322
+ }
1323
+ }, 120);
1324
+ }
1325
+
1326
+ function triggerExampleLoad(idx) {
1327
+ const btnWrap = document.getElementById('example-load-btn');
1328
+ const btn = btnWrap ? (btnWrap.querySelector('button') || btnWrap) : null;
1329
+ if (!btn) return;
1330
+
1331
+ let attempts = 0;
1332
+ function writeIdxAndClick() {
1333
+ attempts += 1;
1334
+ const ok1 = setGradioValue('example-idx-input', String(idx));
1335
+ setGradioValue('example-result-data', '');
1336
+ const currentVal = getValueFromContainer('example-idx-input');
1337
+
1338
+ if (ok1 && currentVal === String(idx)) {
1339
+ btn.click();
1340
+ startExamplePolling();
1341
+ return;
1342
+ }
1343
+
1344
+ if (attempts < 30) {
1345
+ setTimeout(writeIdxAndClick, 100);
1346
+ } else {
1347
+ document.querySelectorAll('.example-card.loading').forEach(c => c.classList.remove('loading'));
1348
+ showToast('Failed to initialize example loader', 'error');
1349
+ }
1350
+ }
1351
+ writeIdxAndClick();
1352
+ }
1353
+
1354
+ document.querySelectorAll('.example-card[data-idx]').forEach(card => {
1355
+ card.addEventListener('click', () => {
1356
+ const idx = card.getAttribute('data-idx');
1357
+ if (!idx) return;
1358
+ document.querySelectorAll('.example-card.loading').forEach(c => c.classList.remove('loading'));
1359
+ card.classList.add('loading');
1360
+ showToast('Loading example...', 'info');
1361
+ triggerExampleLoad(idx);
1362
+ });
1363
+ });
1364
+
1365
+ const observerTarget = document.getElementById('example-result-data');
1366
+ if (observerTarget) {
1367
+ const obs = new MutationObserver(() => {
1368
+ const current = getValueFromContainer('example-result-data');
1369
+ if (!current || current === lastSeenExamplePayload) return;
1370
+ lastSeenExamplePayload = current;
1371
+ if (examplePoller) {
1372
+ clearInterval(examplePoller);
1373
+ examplePoller = null;
1374
+ }
1375
+ applyExamplePayload(current);
1376
+ });
1377
+ obs.observe(observerTarget, {childList:true, subtree:true, characterData:true, attributes:true});
1378
+ }
1379
+
1380
+ if (outputArea) outputArea.value = '';
1381
+ const sb = document.getElementById('sb-run-state');
1382
+ if (sb) sb.textContent = 'Ready';
1383
+ if (imgStatus) imgStatus.textContent = 'No image uploaded';
1384
+ }
1385
+ init();
1386
+ }
1387
+ """
1388
+
1389
+ wire_outputs_js = r"""
1390
+ () => {
1391
+ function watchOutputs() {
1392
+ const resultContainer = document.getElementById('gradio-result');
1393
+ const outArea = document.getElementById('custom-output-textarea');
1394
+
1395
+ if (!resultContainer || !outArea) { setTimeout(watchOutputs, 500); return; }
1396
+
1397
+ let lastText = '';
1398
+
1399
+ function syncOutput() {
1400
+ const el = resultContainer.querySelector('textarea') || resultContainer.querySelector('input');
1401
+ if (!el) return;
1402
+ const val = el.value || '';
1403
+
1404
+ if (val !== lastText) {
1405
+ lastText = val;
1406
+ try {
1407
+ const data = JSON.parse(val);
1408
+ if (data.text !== undefined) {
1409
+ outArea.value = data.text || '';
1410
+ outArea.scrollTop = outArea.scrollHeight;
1411
+ }
1412
+ if (data.annotated && window.__updateAnnotationState) {
1413
+ window.__updateAnnotationState(data.annotated);
1414
+ }
1415
+ if (data.status === 'error') {
1416
+ if (window.__setRunErrorState) window.__setRunErrorState();
1417
+ if (window.__showToast) window.__showToast('Inference failed', 'error');
1418
+ } else if (data.status === 'done') {
1419
+ if (window.__hideLoader) window.__hideLoader();
1420
+ }
1421
+ } catch (e) {
1422
+ outArea.value = val;
1423
+ outArea.scrollTop = outArea.scrollHeight;
1424
+ }
1425
+ }
1426
+ }
1427
+
1428
+ const observer = new MutationObserver(syncOutput);
1429
+ observer.observe(resultContainer, {childList:true, subtree:true, characterData:true, attributes:true});
1430
+ setInterval(syncOutput, 500);
1431
+ }
1432
+ watchOutputs();
1433
  }
 
1434
  """
1435
+
1436
  with gr.Blocks() as demo:
1437
+ hidden_image_b64 = gr.Textbox(value="", elem_id="hidden-image-b64", elem_classes="hidden-input", container=False)
1438
+ prompt = gr.Textbox(value="", elem_id="prompt-gradio-input", elem_classes="hidden-input", container=False)
1439
+ hidden_model_name = gr.Textbox(value="Fara-7B", elem_id="hidden-model-name", elem_classes="hidden-input", container=False)
1440
+ gpu_duration_state = gr.Number(value=60, elem_id="gradio-gpu-duration", elem_classes="hidden-input", container=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1441
 
1442
+ result = gr.Textbox(value="", elem_id="gradio-result", elem_classes="hidden-input", container=False)
 
 
1443
 
1444
+ example_idx = gr.Textbox(value="", elem_id="example-idx-input", elem_classes="hidden-input", container=False)
1445
+ example_result = gr.Textbox(value="", elem_id="example-result-data", elem_classes="hidden-input", container=False)
1446
+ example_load_btn = gr.Button("Load Example", elem_id="example-load-btn")
1447
+
1448
+ gr.HTML(f"""
1449
+ <div class="app-shell">
1450
+ <div class="app-header">
1451
+ <div class="app-header-left">
1452
+ <div class="app-logo">{CUBE_SVG}</div>
1453
+ <span class="app-title">CUA GUI Operator</span>
1454
+ <span class="app-badge">computer use</span>
1455
+ <span class="app-badge fast">visual action grounding</span>
1456
+ </div>
1457
+ </div>
1458
+
1459
+ <div class="model-tabs-bar">
1460
+ {MODEL_TABS_HTML}
1461
+ </div>
1462
+
1463
+ <div class="app-main-row">
1464
+ <div class="app-main-left">
1465
+ <div id="image-drop-zone">
1466
+ <div id="upload-prompt" class="upload-prompt-modern">
1467
+ <div id="upload-click-area" class="upload-click-area">
1468
+ {UPLOAD_PREVIEW_SVG}
1469
+ <span class="upload-main-text">Click or drag a UI screenshot here</span>
1470
+ <span class="upload-sub-text">Upload one interface screenshot for computer-use action localization, click grounding, or agent-style next-step prediction</span>
1471
+ </div>
1472
+ </div>
1473
+
1474
+ <input id="custom-file-input" type="file" accept="image/*" style="display:none;" />
1475
+
1476
+ <div id="single-preview-wrap" class="single-preview-wrap">
1477
+ <div class="single-preview-card">
1478
+ <img id="single-preview-img" src="" alt="Preview">
1479
+ <div class="preview-overlay-actions">
1480
+ <button id="preview-upload-btn" class="preview-action-btn" title="Replace">Upload</button>
1481
+ <button id="preview-clear-btn" class="preview-action-btn" title="Clear">Clear</button>
1482
+ </div>
1483
+ </div>
1484
+ </div>
1485
+ </div>
1486
+
1487
+ <div class="hint-bar">
1488
+ <b>Upload:</b> Click or drag to add a UI image &nbsp;&middot;&nbsp;
1489
+ <b>Model:</b> Switch model tabs from the header &nbsp;&middot;&nbsp;
1490
+ <kbd>Clear</kbd> removes the current image
1491
+ </div>
1492
+
1493
+ <div class="examples-section">
1494
+ <div class="examples-title">Quick Examples</div>
1495
+ <div class="examples-scroll">
1496
+ {EXAMPLE_CARDS_HTML}
1497
+ </div>
1498
+ </div>
1499
+ </div>
1500
+
1501
+ <div class="app-main-right">
1502
+ <div class="panel-card">
1503
+ <div class="panel-card-title">Task Instruction</div>
1504
+ <div class="panel-card-body">
1505
+ <label class="modern-label" for="custom-query-input">Instruction Input</label>
1506
+ <textarea id="custom-query-input" class="modern-textarea" rows="4" placeholder="e.g., click on the search bar, click on the model selector, click on the highlighted button..."></textarea>
1507
+ </div>
1508
+ </div>
1509
+
1510
+ <div style="padding:12px 20px;">
1511
+ <button id="custom-run-btn" class="btn-run">
1512
+ <span id="run-btn-label">Call CUA Agent</span>
1513
+ </button>
1514
+ </div>
1515
+
1516
+ <div class="annot-frame">
1517
+ <div class="annot-title">Visualized Action Points</div>
1518
+ <div class="annot-body">
1519
+ <div id="annotated-output-placeholder" class="annot-placeholder">
1520
+ {ANNOTATION_PLACEHOLDER_SVG}
1521
+ <div class="annot-placeholder-title">Annotated UI preview will appear here</div>
1522
+ <div class="annot-placeholder-sub">Detected click points and grounded actions will be drawn on the uploaded screenshot after inference.</div>
1523
+ </div>
1524
+ <img id="annotated-output-img" src="" alt="Annotated output">
1525
+ </div>
1526
+ </div>
1527
+
1528
+ <div class="output-frame">
1529
+ <div class="out-title">
1530
+ <span id="output-title-label">Agent Model Response</span>
1531
+ <div class="out-title-right">
1532
+ <button id="copy-output-btn" class="out-action-btn" title="Copy">{COPY_SVG} Copy</button>
1533
+ <button id="save-output-btn" class="out-action-btn" title="Save">{SAVE_SVG} Save File</button>
1534
+ </div>
1535
+ </div>
1536
+ <div class="out-body">
1537
+ <div class="modern-loader" id="output-loader">
1538
+ <div class="loader-spinner"></div>
1539
+ <div class="loader-text">Running GUI agent...</div>
1540
+ <div class="loader-bar-track"><div class="loader-bar-fill"></div></div>
1541
+ </div>
1542
+ <div class="output-scroll-wrap">
1543
+ <textarea id="custom-output-textarea" class="output-textarea" placeholder="Agent response will appear here..." readonly></textarea>
1544
+ </div>
1545
+ </div>
1546
+ </div>
1547
+
1548
+ <div class="settings-group">
1549
+ <div class="settings-group-title">Advanced Settings</div>
1550
+ <div class="settings-group-body">
1551
+ <div class="slider-row">
1552
+ <label>GPU Duration (seconds)</label>
1553
+ <input type="range" id="custom-gpu-duration" min="60" max="300" step="30" value="60">
1554
+ <span class="slider-val" id="custom-gpu-duration-val">60</span>
1555
+ </div>
1556
+ </div>
1557
+ </div>
1558
+ </div>
1559
+ </div>
1560
+
1561
+ <div class="exp-note">
1562
+ Experimental GUI Operator Suite &middot; Fara-7B, UI-TARS-1.5-7B, Holo2-4B, ActIO-UI-7B
1563
+ </div>
1564
+
1565
+ <div class="app-statusbar">
1566
+ <div class="sb-section" id="sb-image-status">No image uploaded</div>
1567
+ <div class="sb-section sb-fixed" id="sb-run-state">Ready</div>
1568
+ </div>
1569
+ </div>
1570
+ """)
1571
+
1572
+ run_btn = gr.Button("Run", elem_id="gradio-run-btn")
1573
+
1574
+ demo.load(fn=noop, inputs=None, outputs=None, js=gallery_js)
1575
+ demo.load(fn=noop, inputs=None, outputs=None, js=wire_outputs_js)
1576
+
1577
+ run_btn.click(
1578
+ fn=run_cua,
1579
+ inputs=[
1580
+ hidden_model_name,
1581
+ prompt,
1582
+ hidden_image_b64,
1583
+ gpu_duration_state,
1584
  ],
1585
+ outputs=[result],
1586
+ js=r"""(m, p, img, gd) => {
1587
+ const modelEl = document.querySelector('.model-tab.active');
1588
+ const model = modelEl ? modelEl.getAttribute('data-model') : m;
1589
+ const promptEl = document.getElementById('custom-query-input');
1590
+ const promptVal = promptEl ? promptEl.value : p;
1591
+ const imgContainer = document.getElementById('hidden-image-b64');
1592
+ let imgVal = img;
1593
+ if (imgContainer) {
1594
+ const inner = imgContainer.querySelector('textarea, input');
1595
+ if (inner) imgVal = inner.value;
1596
+ }
1597
+ return [model, promptVal, imgVal, gd];
1598
+ }""",
1599
+ )
1600
+
1601
+ example_load_btn.click(
1602
+ fn=load_example_data,
1603
+ inputs=[example_idx],
1604
+ outputs=[example_result],
1605
+ queue=False,
1606
  )
1607
 
1608
  if __name__ == "__main__":
1609
+ demo.queue(max_size=50).launch(
1610
+ css=css,
1611
+ mcp_server=True,
1612
+ ssr_mode=False,
1613
+ show_error=True,
1614
+ allowed_paths=["examples"],
1615
+ )