Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/game2player: direct link, hf CLI and curl.
- Browser
- Download file 25.7 kB
-
https://huggingface.co/spaces/hugging-apps/game2player/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/game2player/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/game2player/resolve/main/app.py
25.7 kB
| """Game2Player demo: game screenshot + task -> next keyboard/mouse action. | |
| Game2Player (https://arxiv.org/abs/2611.24154) is a visual game-playing agent | |
| fine-tuned from Qwen3.5. This Space runs the released 9B checkpoint with | |
| transformers, replicating the authors' vLLM serving recipe (bf16, max_pixels | |
| 1048576, thinking off, temperature 1.0) and their PlayerBench prompt/decoder. | |
| """ | |
| import json | |
| import math | |
| import re | |
| import time | |
| from pathlib import Path | |
| import spaces # must be imported before torch / transformers | |
| import torch | |
| import gradio as gr | |
| from PIL import Image, ImageDraw | |
| from transformers import AutoProcessor, Qwen3_5ForConditionalGeneration | |
| MODEL_ID = "game2player/Game2Player-9B" | |
| VIEWPORT_W, VIEWPORT_H = 1280, 720 | |
| # Authors' vLLM --mm-processor-kwargs max_pixels=1048576 (min_pixels default 3136). | |
| # New-style processor kwarg: size={'longest_edge': max_pixels, 'shortest_edge': min_pixels}. | |
| IMAGE_SIZE = {"longest_edge": 1048576, "shortest_edge": 3136} | |
| EXAMPLES_DIR = Path(__file__).parent / "examples" | |
| processor = AutoProcessor.from_pretrained(MODEL_ID) | |
| model = Qwen3_5ForConditionalGeneration.from_pretrained( | |
| MODEL_ID, | |
| dtype="auto", # bfloat16 per config.json | |
| attn_implementation="sdpa", | |
| ).to("cuda") | |
| model.eval() | |
| # --------------------------------------------------------------------------- | |
| # Prompt construction, ported 1:1 from the Game2Player repo | |
| # (agents/qwen_prompting.py + catalog/prompt_templates/computer_use_agent_template.j2) | |
| # --------------------------------------------------------------------------- | |
| MEMORY_NOTE = ( | |
| "The user message may contain explicitly labelled previous steps. Treat them " | |
| "as chronological short-term context; act on the labelled current step." | |
| ) | |
| CANONICAL_ACTIONS = ( | |
| "click", "click_hold", "mouse_move", "drag", "scroll", "type", | |
| "press_key", "press_keys", "set_keys", "key_chunk", "wait", | |
| ) | |
| KEY_ALIASES = { | |
| "arrowleft": "ArrowLeft", "left": "ArrowLeft", "arrowright": "ArrowRight", | |
| "right": "ArrowRight", "arrowup": "ArrowUp", "up": "ArrowUp", | |
| "arrowdown": "ArrowDown", "down": "ArrowDown", "space": "Space", | |
| "spacebar": "Space", "enter": "Enter", "return": "Enter", "esc": "Escape", | |
| "escape": "Escape", "tab": "Tab", "backspace": "Backspace", | |
| "delete": "Delete", "del": "Delete", "shift": "Shift", | |
| "shiftleft": "ShiftLeft", "shiftright": "ShiftRight", | |
| "control": "Control", "ctrl": "Control", "controlleft": "ControlLeft", | |
| "controlright": "ControlRight", "alt": "Alt", "altleft": "AltLeft", | |
| "altright": "AltRight", "slash": "/", "period": ".", "comma": ",", | |
| "quote": "'", "apostrophe": "'", "semicolon": ";", "backslash": "\\", | |
| "bracketleft": "[", "bracketright": "]", "minus": "-", "equal": "=", | |
| } | |
| def normalize_key(key): | |
| text = str(key).strip() | |
| if not text: | |
| return None | |
| if len(text) == 1: | |
| return text.lower() | |
| return KEY_ALIASES.get(text.lower(), text) | |
| NOTE_RULE = ( | |
| "In `note`, write one short sentence on why you chose this action; your later " | |
| "steps show it." | |
| ) | |
| KEYBOARD_RULE = ( | |
| "\npress_keys presses its keys simultaneously, then releases them; repeated key " | |
| "names do not create repeated taps. set_keys keeps its keys held until another " | |
| "keyboard action changes them; empty keys releases them. wait advances time " | |
| "without releasing held keys." | |
| ) | |
| TC_OPEN = "<tool" + "_call>" | |
| TC_CLOSE = "</tool" + "_call>" | |
| def build_output_format_block(allowed_keys, thinking=False): | |
| """Port of build_qwen_action_instruction with the repo's ACTION_CONTRACT | |
| (max_actions=1, duration_policy='optional', note=True).""" | |
| keys = sorted({str(k).strip() for k in allowed_keys if str(k).strip()}) | |
| lines = [] | |
| if keys: | |
| lines.extend([ | |
| "- press_key: key, optional duration.", | |
| "- press_keys: keys, optional duration; press simultaneously, then release.", | |
| "- set_keys: keys, optional duration; keep these keys held after the action. Empty keys releases all held keys.", | |
| "- key_chunk: ordered steps containing keys and duration, plus release_after.", | |
| ]) | |
| lines.extend([ | |
| "- type: text.", | |
| "- click: coordinate, optional button (left/right/middle, default left); omit duration.", | |
| "- click_hold: coordinate, optional button and duration.", | |
| "- mouse_move: coordinate.", | |
| "- drag: start_coordinate, end_coordinate, optional button, duration, and steps.", | |
| "- scroll: delta_x and delta_y.", | |
| "- wait: optional duration; advance time without changing held keys.", | |
| "Every action also takes note: one short sentence on why.", | |
| ]) | |
| example_action = ( | |
| {"action": "press_key", "key": keys[0]} if keys | |
| else {"action": "click", "coordinate": [500, 500]} | |
| ) | |
| example_action["note"] = "why this action" | |
| example = json.dumps( | |
| {"name": "computer_use", "arguments": example_action}, | |
| ensure_ascii=False, separators=(",", ":"), | |
| ) | |
| key_rule = ( | |
| "Allowed key values: " + ", ".join(json.dumps(k) for k in keys) + ". Do not use any other value as a key." | |
| if keys else "This game has no keyboard controls; do not emit a keyboard action." | |
| ) | |
| response_rule = ( | |
| "Think in the thinking channel, then emit only the " + TC_OPEN + " as the final " | |
| "answer, with no prose outside the tool-call tags." | |
| if thinking else | |
| "Encode the action plan as one JSON object inside " + TC_OPEN + TC_CLOSE + ". " | |
| "Do not write prose outside the tool-call tags or a Markdown fence." | |
| ) | |
| instructions = ( | |
| "Return exactly one action.\n" | |
| "Duration is optional, in seconds, at most 10. Without a duration, press_key, " | |
| "press_keys and click are 0.05-second taps, and wait, click_hold and drag " | |
| "last the game's default time." | |
| + KEYBOARD_RULE | |
| + "\n" + NOTE_RULE | |
| ) | |
| return "# Computer-use output\n" + instructions + "\n" + response_rule + ( | |
| " Put every action parameter directly inside\n" | |
| "`arguments`; never put function-call syntax inside the `action` string.\n\n" | |
| "Example:\n" + TC_OPEN + example + TC_CLOSE + "\n\n" | |
| "Coordinates are [x, y] in a normalized 0-1000 range mapped to the " | |
| f"{VIEWPORT_W}x{VIEWPORT_H} viewport.\n" | |
| + key_rule + "\n" | |
| "Mouse actions are allowed.\n\n" | |
| "Available actions and fields:\n" + chr(10).join(lines) | |
| ) | |
| def build_system_prompt(game_rules, role_description, control_notes, task, allowed_keys): | |
| role_block = "\n\n".join( | |
| part.strip() for part in (role_description, control_notes, MEMORY_NOTE) if part and part.strip() | |
| ) | |
| return f"""You are an expert game agent. Use the current screenshot and any explicitly | |
| labelled previous steps to choose the next action that best advances the task. | |
| Follow the game rules and control restrictions exactly. | |
| # Game Rules | |
| {game_rules.strip()} | |
| # Role and Controls | |
| {role_block} | |
| # Task Instruction | |
| {task.strip()} | |
| # Output Format | |
| {build_output_format_block(allowed_keys)}""" | |
| # --------------------------------------------------------------------------- | |
| # Output decoding, ported from agents/codecs.py (parse_qwen_action path) | |
| # --------------------------------------------------------------------------- | |
| TOOL_CALL_RE = re.compile( | |
| r"<tool(?:\s+|_)call>\s*(\{[\s\S]*?\})\s*</tool(?:\s+|_)call>", re.IGNORECASE | |
| ) | |
| def _point(value, width, height): | |
| """0-1000 normalized coords -> viewport pixels (agents/codecs._point).""" | |
| if isinstance(value, dict): | |
| x, y = value.get("x"), value.get("y") | |
| elif isinstance(value, (list, tuple)) and len(value) >= 2: | |
| x, y = value[0], value[1] | |
| elif isinstance(value, str): | |
| nums = re.findall(r"-?\d+(?:\.\d+)?", value) | |
| if len(nums) < 2: | |
| return None | |
| x, y = nums[0], nums[1] | |
| else: | |
| return None | |
| try: | |
| x, y = float(x), float(y) | |
| except (TypeError, ValueError): | |
| return None | |
| if not (math.isfinite(x) and math.isfinite(y)): | |
| return None | |
| return ( | |
| max(0.0, min(1000.0, x)) / 1000.0 * width, | |
| max(0.0, min(1000.0, y)) / 1000.0 * height, | |
| ) | |
| def canonicalize_action(args, width=VIEWPORT_W, height=VIEWPORT_H): | |
| """Port of _canonicalize_args (normalized=True) for one action dict.""" | |
| action_type = str(args.get("action") or "").lower() | |
| if action_type not in CANONICAL_ACTIONS: | |
| return None | |
| if action_type in {"click", "click_hold", "mouse_move"}: | |
| point = _point( | |
| args.get("coordinate") or args.get("point") or args.get("start_box") | |
| or ([args.get("x"), args.get("y")] if "x" in args and "y" in args else None), | |
| width, height, | |
| ) | |
| if point is None: | |
| return None | |
| out = {"action": action_type, "x": point[0], "y": point[1]} | |
| if args.get("button") in ("left", "right", "middle"): | |
| out["button"] = args["button"] | |
| return out | |
| if action_type == "drag": | |
| start = _point(args.get("start_coordinate") or args.get("start"), width, height) | |
| end = _point(args.get("end_coordinate") or args.get("end") or args.get("coordinate"), width, height) | |
| if start is None or end is None: | |
| return None | |
| return {"action": "drag", "x1": start[0], "y1": start[1], | |
| "x2": end[0], "y2": end[1]} | |
| if action_type in {"press_key", "press_keys", "set_keys"}: | |
| raw = args.get("keys", args.get("key")) | |
| if isinstance(raw, str): | |
| keys = [k for part in re.split(r"[,+\s]+", raw.strip()) if part for k in [normalize_key(part)] if k] | |
| elif isinstance(raw, (list, tuple)): | |
| keys = [k for k in (normalize_key(i) for i in raw) if k] | |
| else: | |
| keys = [] | |
| if not keys: | |
| return None | |
| if action_type == "press_key": | |
| return {"action": "press_key", "key": keys[0]} | |
| return {"action": action_type, "keys": keys} | |
| if action_type == "type": | |
| text = args.get("text", args.get("content")) | |
| if text in (None, ""): | |
| return None | |
| return {"action": "type", "text": str(text)} | |
| if action_type == "scroll": | |
| def _f(v): | |
| try: | |
| n = float(v) | |
| except (TypeError, ValueError): | |
| return 0.0 | |
| return n if math.isfinite(n) else 0.0 | |
| return {"action": "scroll", "delta_x": _f(args.get("delta_x")), "delta_y": _f(args.get("delta_y"))} | |
| if action_type == "key_chunk": | |
| return None # rare in practice; skip complex chunk parsing in the demo | |
| return {"action": "wait"} if action_type == "wait" else None | |
| def parse_model_output(text): | |
| """Port of parse_qwen_action: tool-call JSON -> canonical action; fall back to raw JSON.""" | |
| for match in TOOL_CALL_RE.finditer(text or ""): | |
| try: | |
| payload = json.loads(match.group(1)) | |
| except json.JSONDecodeError: | |
| continue | |
| if not isinstance(payload, dict): | |
| continue | |
| args = payload.get("arguments") | |
| if isinstance(args, str): | |
| try: | |
| args = json.loads(args) | |
| except json.JSONDecodeError: | |
| continue | |
| if not isinstance(args, dict): | |
| continue | |
| nested = args.get("actions") | |
| candidates = nested if isinstance(nested, list) and nested else [args] | |
| for candidate in candidates: | |
| if not isinstance(candidate, dict): | |
| continue | |
| action = canonicalize_action(candidate) | |
| if action is not None: | |
| note = str( | |
| candidate.get("note") or candidate.get("reasoning") | |
| or candidate.get("reason") or "" | |
| ).strip() | |
| return action, note | |
| final = (text or "").strip() | |
| if final.startswith("```json"): | |
| final = final[len("```json"):].strip() | |
| if final.endswith("```"): | |
| final = final[:-3].strip() | |
| if final.startswith("{"): | |
| try: | |
| payload = json.loads(final) | |
| except json.JSONDecodeError: | |
| payload = None | |
| if isinstance(payload, dict): | |
| args = payload.get("arguments") or payload.get("action") or payload | |
| if isinstance(args, dict): | |
| action = canonicalize_action(args) | |
| if action is not None: | |
| note = str( | |
| args.get("note") or args.get("reasoning") | |
| or args.get("reason") or "" | |
| ).strip() | |
| return action, note | |
| return None, None | |
| # --------------------------------------------------------------------------- | |
| # PlayerBench example presets (games bundled in the Game2Player repo, Apache-2.0) | |
| # --------------------------------------------------------------------------- | |
| # Exact strings from benchmark/playerbench/tasks/{107_simon-sequence,105_memory-match}.yaml | |
| PRESETS = { | |
| "simon": dict( | |
| screenshot=str(EXAMPLES_DIR / "simon_sequence.png"), | |
| task="Repeat Rose Circuit exactly.", | |
| game_rules=( | |
| "You are playing Simon Sequence, a deterministic signal-memory game.\n\n" | |
| "## Objective\n- Repeat the displayed color sequence in exact order.\n\n" | |
| "## Rules\n" | |
| "- The assigned sequence is displayed before input begins and is public to the player.\n" | |
| "- Click one color at a time from left to right.\n" | |
| "- A wrong color fails the attempt; the task ends successfully after the final correct color.\n" | |
| "- The assigned level is selected by the task URL and starts immediately." | |
| ), | |
| role_description="You are repeating the displayed signal. Memorize its order and enter it without mistakes.", | |
| control_notes=( | |
| "- Left-click the color buttons.\n" | |
| "- Follow the numbered signal chips from left to right.\n" | |
| "- The assigned level starts immediately; do not select another level." | |
| ), | |
| allowed_keys="", | |
| ), | |
| "memory": dict( | |
| screenshot=str(EXAMPLES_DIR / "memory_match.png"), | |
| task="Complete Dawn Sprint by finding all three symbol pairs.", | |
| game_rules=( | |
| "You are playing Memory Match, a deterministic offline concentration game. The assigned " | |
| "memory file is already loaded and ready to play.\n\n" | |
| "## Objective\n- Find every matching pair of symbols on the board.\n\n" | |
| "## Rules\n" | |
| "- Click a face-down card to reveal it, then click a second face-down card.\n" | |
| "- Matching symbols stay face up and count as a pair.\n" | |
| "- A mismatch remains visible until you click either revealed card to hide both, then you may continue.\n" | |
| "- The board is deterministic for each URL level and contains no remote assets.\n" | |
| "- Press R or click Restart to reset the assigned memory file." | |
| ), | |
| role_description=( | |
| "You are the memory solver. Reveal cards, remember every visible symbol and its " | |
| "position, then recover all pairs with as few mismatches as possible." | |
| ), | |
| control_notes=( | |
| "- Click the center of any face-down card to reveal it.\n" | |
| "- After a mismatch, click either of the two revealed cards to hide both before selecting another pair.\n" | |
| "- A matched card stays face up and does not need to be clicked again.\n" | |
| "- The assigned level is already open from the URL; do not select another level.\n" | |
| "- R restarts the assigned memory file." | |
| ), | |
| allowed_keys="r", | |
| ), | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Inference | |
| # --------------------------------------------------------------------------- | |
| def annotate(image, action): | |
| """Draw the predicted click/drag point on a copy of the screenshot.""" | |
| canvas = image.copy().convert("RGB") | |
| draw = ImageDraw.Draw(canvas) | |
| if action.get("action") in {"click", "click_hold", "mouse_move"}: | |
| x, y = action.get("x"), action.get("y") | |
| if x is not None and y is not None: | |
| r = 12 | |
| draw.ellipse([x - r, y - r, x + r, y + r], outline=(255, 60, 60), width=4) | |
| draw.line([x - r - 6, y, x + r + 6, y], fill=(255, 60, 60), width=2) | |
| draw.line([x, y - r - 6, x, y + r + 6], fill=(255, 60, 60), width=2) | |
| elif action.get("action") == "drag": | |
| x1, y1, x2, y2 = action.get("x1"), action.get("y1"), action.get("x2"), action.get("y2") | |
| if None not in (x1, y1, x2, y2): | |
| draw.line([x1, y1, x2, y2], fill=(255, 60, 60), width=4) | |
| draw.ellipse([x1 - 10, y1 - 10, x1 + 10, y1 + 10], outline=(255, 60, 60), width=3) | |
| draw.ellipse([x2 - 10, y2 - 10, x2 + 10, y2 + 10], outline=(255, 200, 0), width=3) | |
| return canvas | |
| def describe_action(action, note): | |
| """One-line human rendering of the canonical action.""" | |
| if action is None: | |
| return None | |
| a = action["action"] | |
| if a in {"click", "click_hold", "mouse_move"}: | |
| text = f"{a} at pixel ({int(action['x'])}, {int(action['y'])})" | |
| if a == "click": | |
| text += f" [normalized 0-1000: ({round(action['x'] / VIEWPORT_W * 1000)}, {round(action['y'] / VIEWPORT_H * 1000)})]" | |
| elif a == "drag": | |
| text = f"drag from ({int(action['x1'])}, {int(action['y1'])}) to ({int(action['x2'])}, {int(action['y2'])})" | |
| elif a == "press_key": | |
| text = f"press_key {action['key']}" | |
| elif a in {"press_keys", "set_keys"}: | |
| text = f"{a} {'+'.join(action['keys'])}" | |
| elif a == "type": | |
| text = f'type "{action["text"]}"' | |
| elif a == "scroll": | |
| text = f"scroll dx={action['delta_x']} dy={action['delta_y']}" | |
| else: | |
| text = "wait" | |
| if note: | |
| text += f" — {note}" | |
| return text | |
| def _estimate_duration(screenshot, *args, **kwargs): | |
| """ZeroGPU duration: measured 3.6s at 512 tokens non-thinking; thinking needs more.""" | |
| thinking = args[4] if len(args) > 4 else False | |
| max_new_tokens = args[5] if len(args) > 5 else 512 | |
| if thinking: | |
| return 90 | |
| return min(90, 20 + int(max_new_tokens) * 0.05) | |
| def predict_next_action( | |
| screenshot, | |
| task, | |
| game_rules=PRESETS["simon"]["game_rules"], | |
| role_description=PRESETS["simon"]["role_description"], | |
| control_notes=PRESETS["simon"]["control_notes"], | |
| allowed_keys="", | |
| thinking=False, | |
| max_new_tokens=512, | |
| temperature=1.0, | |
| seed=-1, | |
| progress=gr.Progress(track_tqdm=True), | |
| ): | |
| """Predict the next game action from a screenshot and task description. | |
| Args: | |
| screenshot: current game screenshot (any size; rescaled to ~1M pixels as in training). | |
| task: the task instruction, e.g. "Repeat Warm Signal exactly." | |
| game_rules: the game's rules block (pre-filled from PlayerBench examples). | |
| role_description: the player role description. | |
| control_notes: notes on how controls work. | |
| allowed_keys: comma-separated list of keys the game accepts (empty for mouse-only). | |
| thinking: let the model reason in a thinking channel before the tool call. | |
| max_new_tokens: generation budget. | |
| temperature: sampling temperature. | |
| seed: random seed; -1 keeps sampling random. | |
| Returns: | |
| (annotated screenshot, action summary, raw model output, elapsed seconds). | |
| """ | |
| if screenshot is None: | |
| raise gr.Error("Please provide a game screenshot first.") | |
| # API callers may omit the preset fields (None) — fall back to the Simon preset. | |
| if not game_rules: | |
| game_rules = PRESETS["simon"]["game_rules"] | |
| if role_description is None: | |
| role_description = PRESETS["simon"]["role_description"] | |
| if control_notes is None: | |
| control_notes = PRESETS["simon"]["control_notes"] | |
| if task is None: | |
| task = PRESETS["simon"]["task"] | |
| if isinstance(screenshot, str): | |
| image = Image.open(screenshot).convert("RGB") | |
| else: | |
| image = screenshot.convert("RGB") | |
| keys = [k.strip() for k in re.split(r"[,+\s]+", (allowed_keys or "").strip()) if k.strip()] | |
| system_prompt = build_system_prompt(game_rules, role_description, control_notes, task, keys) | |
| messages = [ | |
| {"role": "system", "content": [{"type": "text", "text": system_prompt}]}, | |
| {"role": "user", "content": [ | |
| {"type": "text", "text": "# Current step\nCurrent game screen:\n"}, | |
| {"type": "image", "image": image}, | |
| {"type": "text", "text": "\nReturn the next action."}, | |
| ]}, | |
| ] | |
| torch.manual_seed(seed if seed is not None and seed >= 0 else int(time.time())) | |
| inputs = processor.apply_chat_template( | |
| messages, | |
| tokenize=True, | |
| add_generation_prompt=True, | |
| return_dict=True, | |
| return_tensors="pt", | |
| enable_thinking=bool(thinking), | |
| processor_kwargs={"size": IMAGE_SIZE}, | |
| ).to("cuda") | |
| start = time.perf_counter() | |
| with torch.inference_mode(): | |
| generated = model.generate( | |
| **inputs, | |
| max_new_tokens=int(max_new_tokens), | |
| temperature=float(temperature), | |
| do_sample=float(temperature) > 0, | |
| top_p=1.0, | |
| ) | |
| elapsed = time.perf_counter() - start | |
| trimmed = generated[0][inputs["input_ids"].shape[1]:] | |
| raw = processor.decode(trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False) | |
| action, note = parse_model_output(raw) | |
| summary = describe_action(action, note) | |
| if summary is None: | |
| summary = "Could not parse an action from the model output — see the raw output below." | |
| annotated = annotate(image, action) if action and action.get("action") in { | |
| "click", "click_hold", "mouse_move", "drag", | |
| } else image | |
| return annotated, summary, raw, f"{elapsed:.1f}s" | |
| with gr.Blocks(title="Game2Player — Visual Game-Playing Agent") as demo: | |
| gr.Markdown( | |
| f""" | |
| # 🎮 Game2Player — Visual Game-Playing Agent | |
| **Game2Player: Scaling Gameplay Experience for Visual Agents through Environment-Player Co-Development** | |
| ([paper](https://arxiv.org/abs/2611.24154) · [code](https://github.com/game2player/Game2Player) · [model card](https://huggingface.co/{MODEL_ID})) | |
| This demo runs the released **Game2Player-9B** checkpoint (Qwen3.5-9B fine-tune) with transformers, | |
| reproducing the authors' serving recipe (bf16, ≤1M pixels per screenshot, temperature 1.0). | |
| Give it a **game screenshot** and the **task + rules**, and it predicts the next | |
| keyboard or mouse action as a `computer_use` tool call. | |
| The two presets are real tasks from **PlayerBench**, the paper's benchmark — screenshots | |
| captured from the benchmark's bundled games (`107_simon-sequence`, `105_memory-match`, Apache-2.0). | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| screenshot = gr.Image( | |
| type="pil", label="Game screenshot", height=420, | |
| sources=["upload", "clipboard"], | |
| ) | |
| task = gr.Textbox( | |
| label="Task instruction", lines=1, | |
| value="Repeat Rose Circuit exactly.", | |
| ) | |
| with gr.Accordion("Game rules and controls (advanced)", open=False): | |
| game_rules = gr.Textbox(label="Game rules", lines=8, value=PRESETS["simon"]["game_rules"]) | |
| role_description = gr.Textbox(label="Role description", lines=2, value=PRESETS["simon"]["role_description"]) | |
| control_notes = gr.Textbox(label="Control notes", lines=3, value=PRESETS["simon"]["control_notes"]) | |
| allowed_keys = gr.Textbox(label="Allowed keys (comma-separated; empty = mouse-only)", value="") | |
| with gr.Accordion("Generation settings", open=False): | |
| thinking = gr.Checkbox(label="Enable thinking channel", value=False) | |
| max_new_tokens = gr.Slider(64, 2048, value=512, step=64, label="Max new tokens") | |
| temperature = gr.Slider(0.0, 2.0, value=1.0, step=0.1, label="Temperature") | |
| seed = gr.Number(value=-1, label="Seed (-1 = random)") | |
| predict_btn = gr.Button("Predict next action", variant="primary") | |
| with gr.Column(): | |
| annotated_out = gr.Image(type="pil", label="Predicted action on screen", height=420) | |
| action_out = gr.Textbox(label="Next action", lines=2) | |
| raw_out = gr.Textbox(label="Raw model output", lines=6) | |
| elapsed_out = gr.Textbox(label="Inference time", show_label=False) | |
| predict_btn.click( | |
| predict_next_action, | |
| inputs=[screenshot, task, game_rules, role_description, control_notes, | |
| allowed_keys, thinking, max_new_tokens, temperature, seed], | |
| outputs=[annotated_out, action_out, raw_out, elapsed_out], | |
| api_name="predict_next_action", | |
| ) | |
| example_inputs = [screenshot, task, game_rules, role_description, control_notes, allowed_keys] | |
| gr.Examples( | |
| examples=[ | |
| [p["screenshot"], p["task"], p["game_rules"], p["role_description"], | |
| p["control_notes"], p["allowed_keys"]] | |
| for p in (PRESETS["simon"], PRESETS["memory"]) | |
| ], | |
| inputs=example_inputs, | |
| fn=predict_next_action, | |
| outputs=[annotated_out, action_out, raw_out, elapsed_out], | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="PlayerBench examples (real tasks from the paper's benchmark)", | |
| ) | |
| demo.launch(mcp_server=True, theme=gr.themes.Citrus()) | |