"""SeekUI behind one call: a screen and the text of a target in, a predicted visual-search scanpath out. This is the Hugging Face Inference Endpoints custom handler (`EndpointHandler`, the file at the root of the model's repository). serve.py runs the same class on a machine of ours, so a live frame and a historical frame are predicted by the same code wherever it runs. The prompt, the greedy decoding and the 512-token limit are those of the authors' release: Guo, Jiang, Leiva and Oulasvirta, "SeekUI", CHI 2026, https://doi.org/10.1145/3772318.3791178. The fixations are a model's guess of how people search a screen. They are not an observation of anyone. """ import base64 import io import pathlib import re import time import torch from PIL import Image from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration # The authors' weights, at the revision measured. A directory that holds a checkpoint (serve.py's) is used instead. MODEL = "sushizixin1/SeekUI" REVISION = "f776d40e4beaf7f7fb4f00abcbcfa717e7ae5039" MAX_NEW_TOKENS = 512 MAX_TARGET = 200 PROMPT = ( "Given the image with width {width} and height {height}, " "what is the scanpath for the visual search task on this GUI? " 'The text on the target element is "{target}".\n' "Output the thinking process in and final answer in tags. " "The output answer format should be as follows:\n" " ... The scanpath is [x1, y1] [x2, y2] ...\n" "Please strictly follow the format.\n" ) def parse(content, width, height): """The model's text as (explanation, fixations). A point outside the image is dropped and counted, never moved.""" thought = re.search(r"(.*?)", content, re.S) answer = re.search(r"(.*?)", content, re.S) text = answer.group(1) if answer else content points = [(int(x), int(y)) for x, y in re.findall(r"\[(\d+)\s*[\s,]\s*(\d+)\]", text)] inside = [{"x": x, "y": y} for x, y in points if x < width and y < height] return (thought.group(1).strip() if thought else ""), inside, len(points) - len(inside) class EndpointHandler: def __init__(self, path=""): self.device = "cuda" if torch.cuda.is_available() else "cpu" dtype = torch.bfloat16 if self.device == "cuda" else torch.float32 local = (pathlib.Path(path) / "model.safetensors.index.json").exists() source = {"pretrained_model_name_or_path": path if local else MODEL, "revision": None if local else REVISION} self.processor = AutoProcessor.from_pretrained(**source) self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(**source, dtype=dtype).to(self.device).eval() def __call__(self, data): given = data.get("inputs", data) target = str(given.get("target", "")).strip() if not target or len(target) > MAX_TARGET: return {"error": f"target is the text of the element searched for, 1 to {MAX_TARGET} characters"} image = Image.open(io.BytesIO(base64.b64decode(given["image"]))).convert("RGB") width, height = image.size started = time.perf_counter() messages = [{"role": "user", "content": [ {"type": "text", "text": PROMPT.format(width=width, height=height, target=target)}, {"type": "image", "image": image}, ]}] text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) inputs = self.processor(text=[text], images=[image], padding=True, return_tensors="pt").to(self.device) with torch.inference_mode(): out = self.model.generate(**inputs, max_new_tokens=MAX_NEW_TOKENS, do_sample=False) fresh = out[0][inputs.input_ids.shape[1]:] content = self.processor.decode(fresh, skip_special_tokens=True, clean_up_tokenization_spaces=False) explanation, fixations, outside = parse(content, width, height) return { "width": width, "height": height, "target": target, "fixations": fixations, "outside": outside, "explanation": explanation, "seconds": round(time.perf_counter() - started, 3), "tokens_in": int(inputs.input_ids.shape[1]), "tokens_out": int(fresh.shape[0]), }