routestick patternlock text update

gradio-web/config.py

@@ -37,7 +37,7 @@ DEMO_VIDEO_ENV_IDS = [
 
 UI_TEXT = {
     "log": {
-        "action_selection_prompt": "please select the action below 👇🏻,\nsome actions also need to select keypoint",
+        "action_selection_prompt": "please select the action in the right 👈,\nsome actions also need to select keypoint",
         "demo_video_prompt": 'press "Watch Video Input🎬" to watch a video\nNote: you can only watch the video once',
         "session_error": "Session Error",
         "reference_action_error": "Ground Truth Action Error: {error}",
@@ -53,6 +53,9 @@ UI_TEXT = {
         "select_keypoint": "please click the keypoint selection image",
         "select_keypoint_before_execute": "please click the keypoint selection image before execute!",
     },
+    "actions": {
+        "keypoint_required_suffix": " (click mouse 🖱️ to select 🎯)",
+    },
     "errors": {
         "load_missing_task": "Error loading task: missing current_task",
         "load_invalid_task": "Error loading task: invalid task payload",
@@ -66,6 +69,45 @@ UI_TEXT = {
     },
 }
 
+UI_ACTION_TEXT_OVERRIDES = {
+    "PatternLock": {
+        "move forward": "move forward↓",
+        "move backward": "move backward↑",
+        "move left": "move left→",
+        "move right": "move right←",
+        "move forward-left": "move forward-left↘︎",
+        "move forward-right": "move forward-right↙︎",
+        "move backward-left": "move backward-left↗︎",
+        "move backward-right": "move backward-right↖︎",
+    },
+    "RouteStick": {
+        "move to the nearest left target by circling around the stick clockwise": "move left clockwise↘︎→↗︎ ◟→◞",
+        "move to the nearest right target by circling around the stick clockwise": "move right clockwise↖︎←↙︎ ◟←◞",
+        "move to the nearest left target by circling around the stick counterclockwise": "move left counterclockwise↗︎→↘︎ ◜→◝",
+        "move to the nearest right target by circling around the stick counterclockwise": "move right counterclockwise↙︎←↖︎ ◜←◝",
+    },
+}
+
+ROUTESTICK_OVERLAY_ACTION_TEXTS = [
+    "move to the nearest left target by circling around the stick clockwise",
+    "move to the nearest left target by circling around the stick counterclockwise",
+    "move to the nearest right target by circling around the stick clockwise",
+    "move to the nearest right target by circling around the stick counterclockwise",
+]
+
+
+def get_ui_action_text(env_id, action_text):
+    """
+    Return display-only action text overrides for a specific env/action pair.
+    Falls back to the original action text when no override is configured.
+    """
+    if not isinstance(action_text, str):
+        return action_text
+    if not isinstance(env_id, str) or not env_id:
+        return action_text
+    env_overrides = UI_ACTION_TEXT_OVERRIDES.get(env_id, {})
+    return env_overrides.get(action_text, action_text)
+
 def should_show_demo_video(env_id):
     """
     判断指定的环境ID是否应该显示demonstration video

gradio-web/gradio_callbacks.py

@@ -38,6 +38,7 @@ from config import (
     SESSION_TIMEOUT,
     UI_TEXT,
     USE_SEGMENTED_VIEW,
+    get_ui_action_text,
     should_show_demo_video,
 )
 from process_session import ScrewPlanFailureError, ProcessSessionProxy
@@ -105,17 +106,34 @@ def get_videoplacebutton_goal(original_goal: str) -> str:
 def _ui_option_label(session, opt_label: str, opt_idx: int) -> str:
     """
     仅在 Gradio UI 层对选项显示文案做覆盖（不改底层 env/options 生成逻辑）。
-    目前只对 RouteStick 任务把 4 个长句 label 显示为短 label。
+    优先使用 raw_solve_options 中的原始 label/action 重新组装显示文本，
+    并按 env_id 做 display-only action 文案映射。
     """
-    env_id = getattr(session, "env_id", None)
-    if env_id == "RouteStick":
-        routestick_map = {
-            0: "move left clockwise",
-            1: "move right clockwise",
-            2: "move left counterclockwise",
-            3: "move right counterclockwise",
-        }
-        return routestick_map.get(int(opt_idx), opt_label)
+    try:
+        option_index = int(opt_idx)
+    except (TypeError, ValueError):
+        return opt_label
+
+    raw_solve_options = getattr(session, "raw_solve_options", None)
+    if not isinstance(raw_solve_options, list):
+        return opt_label
+    if not (0 <= option_index < len(raw_solve_options)):
+        return opt_label
+
+    raw_option = raw_solve_options[option_index]
+    if not isinstance(raw_option, dict):
+        return opt_label
+
+    raw_label = str(raw_option.get("label", "")).strip()
+    raw_action = str(raw_option.get("action", "")).strip()
+    mapped_action = get_ui_action_text(getattr(session, "env_id", None), raw_action)
+
+    if raw_label and mapped_action:
+        return f"{raw_label}. {mapped_action}"
+    if mapped_action:
+        return mapped_action
+    if raw_label:
+        return raw_label
     return opt_label
 
 
@@ -508,7 +526,7 @@ def _load_status_task(uid, status):
         if 0 <= opt_idx < len(session.raw_solve_options):
             opt = session.raw_solve_options[opt_idx]
             if opt.get("available"):
-                opt_label_with_hint = f"{opt_label} (click mouse 🖱️ to select 🎯)"
+                opt_label_with_hint = f"{opt_label}{_ui_text('actions', 'keypoint_required_suffix')}"
             else:
                 opt_label_with_hint = opt_label
         else:
@@ -860,6 +878,7 @@ def on_reference_action(uid):
     option_idx = reference.get("option_idx")
     option_label = str(reference.get("option_label", "")).strip()
     option_action = str(reference.get("option_action", "")).strip()
+    option_action = get_ui_action_text(getattr(session, "env_id", None), option_action)
     need_coords = bool(reference.get("need_coords", False))
     coords_xy = reference.get("coords_xy")
 

gradio-web/image_utils.py

@@ -10,7 +10,7 @@ import math
 from pathlib import Path
 from PIL import Image, ImageDraw, ImageFont
 import cv2
-from config import VIDEO_PLAYBACK_FPS
+from config import VIDEO_PLAYBACK_FPS, ROUTESTICK_OVERLAY_ACTION_TEXTS, get_ui_action_text
 
 # DEPRECATED: 历史任务特化图像叠加配置，保留仅为兼容旧代码路径。
 # 当前已统一关闭任务特化渲染。
@@ -354,6 +354,11 @@ def draw_coordinate_axes(img, position="right", rotate_180=False, env_id=None):
     
     # 如果是 RouteStick 任务，绘制旋转方向示意图（左侧或右侧）
     if env_id == "RouteStick" and (position == "right" or position == "left"):
+        lcw_text, lccw_text, rcw_text, rccw_text = [
+            get_ui_action_text("RouteStick", action_text)
+            for action_text in ROUTESTICK_OVERLAY_ACTION_TEXTS
+        ]
+
         # 绘制四个半圆箭头示意图（垂直排列）
         # 示意图位置：在图像的左侧或右侧，从上到下垂直排列
         illustration_width = 220  # 示意图区域宽度（已弃用，保留以保持兼容性）
@@ -401,7 +406,6 @@ def draw_coordinate_axes(img, position="right", rotate_180=False, env_id=None):
         draw_semicircle(draw, (lcw_center_x , lcw_center_y+15), semicircle_radius, line_color, line_width, half="upper", start_pos="left", end_pos="right", arrow_position="end", arrow_size=arrow_size)
         
         # 添加标签 "L CW"
-        lcw_text = "Left Clockwise"
         lcw_bbox = draw.textbbox((0, 0), lcw_text, font=small_font)
         lcw_text_width = lcw_bbox[2] - lcw_bbox[0]
         lcw_text_height = lcw_bbox[3] - lcw_bbox[1]
@@ -418,7 +422,6 @@ def draw_coordinate_axes(img, position="right", rotate_180=False, env_id=None):
         draw_semicircle(draw, (lccw_center_x, lccw_center_y), semicircle_radius, line_color, line_width, half="lower", start_pos="left", end_pos="right", arrow_position="end", arrow_size=arrow_size)
 
         # 添加标签 "L CCW"
-        lccw_text = "Left Counterclockwise"
         lccw_bbox = draw.textbbox((0, 0), lccw_text, font=small_font)
         lccw_text_width = lccw_bbox[2] - lccw_bbox[0]
         lccw_text_height = lccw_bbox[3] - lccw_bbox[1]
@@ -435,7 +438,6 @@ def draw_coordinate_axes(img, position="right", rotate_180=False, env_id=None):
         draw_semicircle(draw, (rcw_center_x , rcw_center_y), semicircle_radius, line_color, line_width, half="lower", start_pos="right", end_pos="left", arrow_position="end", arrow_size=arrow_size)
 
         # 添加标签 "R CW"
-        rcw_text = "Right Clockwise"
         rcw_bbox = draw.textbbox((0, 0), rcw_text, font=small_font)
         rcw_text_width = rcw_bbox[2] - rcw_bbox[0]
         rcw_text_height = rcw_bbox[3] - rcw_bbox[1]
@@ -452,7 +454,6 @@ def draw_coordinate_axes(img, position="right", rotate_180=False, env_id=None):
         draw_semicircle(draw, (rccw_center_x , rccw_center_y+15), semicircle_radius, line_color, line_width, half="upper",start_pos="right", end_pos="left", arrow_position="end", arrow_size=arrow_size)
 
         # 添加标签 "R CCW"
-        rccw_text = "Right Counterclockwise"
         rccw_bbox = draw.textbbox((0, 0), rccw_text, font=small_font)
         rccw_text_width = rccw_bbox[2] - rccw_bbox[0]
         rccw_text_height = rccw_bbox[3] - rccw_bbox[1]

gradio-web/main.py

@@ -6,6 +6,9 @@ import sys
 import tempfile
 from pathlib import Path
 
+
+
+
 APP_DIR = Path(__file__).resolve().parent
 PROJECT_ROOT = APP_DIR.parent
 SRC_DIR = PROJECT_ROOT / "src"
@@ -13,6 +16,17 @@ VIDEOS_DIR = APP_DIR / "videos"
 TEMP_DEMOS_DIR = PROJECT_ROOT / "temp_demos"
 CWD_TEMP_DEMOS_DIR = Path.cwd() / "temp_demos"
 
+
+def configure_runtime_devices():
+    """Restrict the app to physical GPU 1 and map rendering to the visible device."""
+    os.environ["CUDA_VISIBLE_DEVICES"] = "1"
+    os.environ.setdefault("NVIDIA_VISIBLE_DEVICES", "1")
+    # After masking to physical GPU 1, libraries should use logical cuda:0.
+    os.environ["SAPIEN_RENDER_DEVICE"] = "cuda:0"
+
+
+configure_runtime_devices()
+
 if str(PROJECT_ROOT) not in sys.path:
     sys.path.insert(0, str(PROJECT_ROOT))
 if str(SRC_DIR) not in sys.path:

gradio-web/test/test_reference_action_callbacks.py

@@ -4,8 +4,9 @@ from PIL import Image
 
 
 class _FakeSession:
-    def __init__(self, reference_payload):
+    def __init__(self, reference_payload, env_id="BinFill"):
         self._reference_payload = reference_payload
+        self.env_id = env_id
 
     def get_reference_action(self):
         return self._reference_payload
@@ -92,3 +93,32 @@ def test_on_option_select_keeps_valid_coords_when_option_needs_coords(monkeypatc
 
     assert coords_text == "12, 34"
     assert img_update.get("interactive") is True
+
+
+def test_on_reference_action_uses_configured_action_text_override(monkeypatch, reload_module):
+    config = reload_module("config")
+    callbacks = reload_module("gradio_callbacks")
+
+    session = _FakeSession(
+        {
+            "ok": True,
+            "option_idx": 0,
+            "option_label": "a",
+            "option_action": "move forward",
+            "need_coords": False,
+            "coords_xy": None,
+            "message": "ok",
+        },
+        env_id="PatternLock",
+    )
+
+    monkeypatch.setattr(callbacks, "update_session_activity", lambda uid: None)
+    monkeypatch.setattr(callbacks, "get_session", lambda uid: session)
+
+    _img, _option_update, coords_text, log_html = callbacks.on_reference_action("uid-1")
+
+    assert coords_text == config.UI_TEXT["coords"]["not_needed"]
+    assert log_html == config.UI_TEXT["log"]["reference_action_message"].format(
+        option_label="a",
+        option_action="move forward↑",
+    )

gradio-web/test/test_ui_text_config.py

@@ -4,8 +4,25 @@ import pytest
 
 
 class _FakeOptionSession:
-    def __init__(self):
-        self.raw_solve_options = [{"available": True}]
+    def __init__(self, env_id="BinFill", raw_solve_options=None):
+        self.env_id = env_id
+        self.raw_solve_options = raw_solve_options or [{"available": True}]
+
+
+class _FakeLoadSession:
+    def __init__(self, env_id, available_options, raw_solve_options):
+        self.env_id = env_id
+        self.available_options = available_options
+        self.raw_solve_options = raw_solve_options
+        self.language_goal = ""
+        self.demonstration_frames = []
+
+    def load_episode(self, env_id, episode_idx):
+        self.env_id = env_id
+        return "IMG", f"loaded {env_id} {episode_idx}"
+
+    def get_pil_image(self, use_segmented=False):
+        return "IMG"
 
 
 def test_on_option_select_uses_configured_select_keypoint_message(monkeypatch, reload_module):
@@ -67,3 +84,111 @@ def test_missing_session_paths_use_configured_session_error(monkeypatch, reload_
     assert log_text == "Session Error From Config"
     assert map_img is None
     assert map_text == "Session Error From Config"
+
+
+def test_get_ui_action_text_uses_configured_overrides_and_fallback(reload_module):
+    config = reload_module("config")
+
+    patternlock_expected = {
+        "move forward": "move forward↑",
+        "move backward": "move backward↓",
+        "move left": "move left→",
+        "move right": "move right←",
+        "move forward-left": "move forward-left↘︎",
+        "move forward-right": "move forward-right↙︎",
+        "move backward-left": "move backward-left↗︎",
+        "move backward-right": "move backward-right↖︎",
+    }
+    routestick_expected = {
+        "move to the nearest left target by circling around the stick clockwise": "move left clockwise↘︎→↗︎",
+        "move to the nearest right target by circling around the stick clockwise": "move right clockwise↖︎←↙︎",
+        "move to the nearest left target by circling around the stick counterclockwise": "move left counterclockwise↗︎→↘︎",
+        "move to the nearest right target by circling around the stick counterclockwise": "move right counterclockwise↖︎←↙︎",
+    }
+
+    for raw_action, expected in patternlock_expected.items():
+        assert config.get_ui_action_text("PatternLock", raw_action) == expected
+    for raw_action, expected in routestick_expected.items():
+        assert config.get_ui_action_text("RouteStick", raw_action) == expected
+    assert config.get_ui_action_text("BinFill", "pick up the cube") == "pick up the cube"
+
+
+def test_ui_option_label_uses_patternlock_configured_action_text(reload_module):
+    reload_module("config")
+    callbacks = reload_module("gradio_callbacks")
+    session = _FakeOptionSession(
+        env_id="PatternLock",
+        raw_solve_options=[{"label": "a", "action": "move forward", "available": False}],
+    )
+
+    assert callbacks._ui_option_label(session, "fallback", 0) == "a. move forward↑"
+
+
+def test_ui_option_label_uses_routestick_configured_action_text(reload_module):
+    reload_module("config")
+    callbacks = reload_module("gradio_callbacks")
+    session = _FakeOptionSession(
+        env_id="RouteStick",
+        raw_solve_options=[
+            {
+                "label": "d",
+                "action": "move to the nearest right target by circling around the stick counterclockwise",
+                "available": False,
+            }
+        ],
+    )
+
+    assert callbacks._ui_option_label(session, "fallback", 0) == "d. move right counterclockwise↖︎←↙︎"
+
+
+def test_load_status_task_appends_configured_keypoint_suffix_after_mapped_label(monkeypatch, reload_module):
+    config = reload_module("config")
+    callbacks = reload_module("gradio_callbacks")
+    session = _FakeLoadSession(
+        env_id="PatternLock",
+        available_options=[("a. move forward", 0)],
+        raw_solve_options=[{"label": "a", "action": "move forward", "available": [object()]}],
+    )
+
+    monkeypatch.setattr(callbacks, "get_session", lambda uid: session)
+    monkeypatch.setattr(callbacks, "reset_play_button_clicked", lambda uid: None)
+    monkeypatch.setattr(callbacks, "reset_execute_count", lambda uid, env_id, episode_idx: None)
+    monkeypatch.setattr(callbacks, "set_task_start_time", lambda uid, env_id, episode_idx, start_time: None)
+    monkeypatch.setattr(callbacks, "set_ui_phase", lambda uid, phase: None)
+    monkeypatch.setattr(callbacks, "get_task_hint", lambda env_id: "")
+    monkeypatch.setattr(callbacks, "should_show_demo_video", lambda env_id: False)
+
+    result = callbacks._load_status_task(
+        "uid-1",
+        {"current_task": {"env_id": "PatternLock", "episode_idx": 1}, "completed_count": 3},
+    )
+
+    assert result[4]["choices"] == [
+        (
+            f"a. move forward↑{config.UI_TEXT['actions']['keypoint_required_suffix']}",
+            0,
+        )
+    ]
+
+
+def test_draw_coordinate_axes_uses_configured_routestick_overlay_labels(monkeypatch, reload_module):
+    config = reload_module("config")
+    image_utils = reload_module("image_utils")
+    recorded_texts = []
+    original_text = image_utils.ImageDraw.ImageDraw.text
+
+    def _record_text(self, xy, text, *args, **kwargs):
+        recorded_texts.append(text)
+        return original_text(self, xy, text, *args, **kwargs)
+
+    monkeypatch.setattr(image_utils.ImageDraw.ImageDraw, "text", _record_text)
+
+    img = image_utils.Image.new("RGB", (220, 260), color=(0, 0, 0))
+    image_utils.draw_coordinate_axes(img, position="left", env_id="RouteStick")
+
+    expected_labels = [
+        config.get_ui_action_text("RouteStick", action_text)
+        for action_text in config.ROUTESTICK_OVERLAY_ACTION_TEXTS
+    ]
+    for label in expected_labels:
+        assert label in recorded_texts