highlight 1

gradio-web/config.py

@@ -41,6 +41,7 @@ DEMO_VIDEO_ENV_IDS = [
 UI_TEXT = {
     "log": {
         "action_selection_prompt": "please select the action in the left 👈,\nsome actions also need to select keypoint",
+        "keypoint_selection_prompt": "current action needs location input, please click on the image to select key pixel",
         "demo_video_prompt": 'press "Watch Video Input🎬" to watch a video\nNote: you can only watch the video once',
         "session_error": "Session Error",
         "reference_action_error": "Ground Truth Action Error: {error}",
@@ -98,6 +99,16 @@ ROUTESTICK_OVERLAY_ACTION_TEXTS = [
     "move to the nearest right target by circling around the stick counterclockwise",
 ]
 
+LIVE_OBS_BASE_CLASS = "live-obs-resizable"
+LIVE_OBS_KEYPOINT_WAIT_CLASS = "live-obs-keypoint-waiting"
+
+
+def get_live_obs_elem_classes(waiting_for_keypoint=False):
+    classes = [LIVE_OBS_BASE_CLASS]
+    if waiting_for_keypoint:
+        classes.append(LIVE_OBS_KEYPOINT_WAIT_CLASS)
+    return classes
+
 
 def get_ui_action_text(env_id, action_text):
     """

gradio-web/gradio_callbacks.py

@@ -40,6 +40,7 @@ from config import (
     SESSION_TIMEOUT,
     UI_TEXT,
     USE_SEGMENTED_VIEW,
+    get_live_obs_elem_classes,
     get_ui_action_text,
     should_show_demo_video,
 )
@@ -59,6 +60,55 @@ def _ui_text(section, key, **kwargs):
     return template.format(**kwargs) if kwargs else template
 
 
+_LIVE_OBS_UPDATE_SKIP = object()
+
+
+def _action_selection_log():
+    return format_log_markdown(_ui_text("log", "action_selection_prompt"))
+
+
+def _keypoint_selection_log():
+    return format_log_markdown(_ui_text("log", "keypoint_selection_prompt"))
+
+
+def _live_obs_update(
+    *,
+    value=_LIVE_OBS_UPDATE_SKIP,
+    interactive=None,
+    visible=None,
+    waiting_for_keypoint=False,
+):
+    kwargs = {
+        "elem_classes": get_live_obs_elem_classes(waiting_for_keypoint=waiting_for_keypoint),
+    }
+    if value is not _LIVE_OBS_UPDATE_SKIP:
+        kwargs["value"] = value
+    if interactive is not None:
+        kwargs["interactive"] = interactive
+    if visible is not None:
+        kwargs["visible"] = visible
+    return gr.update(**kwargs)
+
+
+def _parse_option_idx(option_value):
+    if isinstance(option_value, tuple):
+        _, option_idx = option_value
+        return option_idx
+    return option_value
+
+
+def _option_requires_coords(session, option_value) -> bool:
+    option_idx = _parse_option_idx(option_value)
+    if not isinstance(option_idx, int):
+        return False
+    raw_solve_options = getattr(session, "raw_solve_options", None)
+    if not isinstance(raw_solve_options, list):
+        return False
+    if not (0 <= option_idx < len(raw_solve_options)):
+        return False
+    return bool(raw_solve_options[option_idx].get("available"))
+
+
 def _should_enqueue_sample(sample_index: int) -> bool:
     factor = max(1, int(KEYFRAME_DOWNSAMPLE_FACTOR))
     return sample_index % factor == 0
@@ -224,7 +274,7 @@ def on_video_end(uid):
     Called when the demonstration video finishes playing.
     Updates the system log to prompt for action selection.
     """
-    return format_log_markdown(_ui_text("log", "action_selection_prompt"))
+    return _action_selection_log()
 
 
 def on_demo_video_play(uid):
@@ -263,7 +313,7 @@ def switch_to_execute_phase(uid):
         gr.update(interactive=False),  # exec_btn
         gr.update(interactive=False),  # restart_episode_btn
         gr.update(interactive=False),  # next_task_btn
-        gr.update(interactive=False),  # img_display
+        _live_obs_update(interactive=False),  # img_display
         gr.update(interactive=False),  # reference_action_btn
     )
 
@@ -279,7 +329,7 @@ def switch_to_action_phase(uid=None):
         gr.update(),  # exec_btn (keep execute_step result)
         gr.update(),  # restart_episode_btn (keep execute_step result)
         gr.update(),  # next_task_btn (keep execute_step result)
-        gr.update(interactive=True),  # img_display
+        _live_obs_update(interactive=True),  # img_display
         gr.update(interactive=True),  # reference_action_btn
     )
 
@@ -414,7 +464,7 @@ def refresh_live_obs(uid, ui_phase):
     img = _prepare_refresh_frame(latest)
     if img is None:
         return gr.update()
-    return gr.update(value=img, interactive=False)
+    return _live_obs_update(value=img, interactive=False)
 
 
 def on_video_end_transition(uid):
@@ -423,7 +473,7 @@ def on_video_end_transition(uid):
         gr.update(visible=False),  # video_phase_group
         gr.update(visible=True),   # action_phase_group
         gr.update(visible=True),   # control_panel_group
-        format_log_markdown(_ui_text("log", "action_selection_prompt")),
+        _action_selection_log(),
         gr.update(visible=False, interactive=False),  # watch_demo_video_btn
     )
 
@@ -433,7 +483,7 @@ def _task_load_failed_response(uid, message):
     return (
         uid,
         gr.update(visible=True),  # main_interface
-        gr.update(value=None, interactive=False),  # img_display
+        _live_obs_update(value=None, interactive=False),  # img_display
         format_log_markdown(message),  # log_output
         gr.update(choices=[], value=None),  # options_radio
         "",  # goal_box
@@ -514,7 +564,7 @@ def _load_status_task(uid, status):
         return (
             uid,
             gr.update(visible=True),  # main_interface
-            gr.update(value=None, interactive=False),  # img_display
+            _live_obs_update(value=None, interactive=False),  # img_display
             format_log_markdown(_ui_text("errors", "load_episode_error", load_msg=load_msg)),  # log_output
             gr.update(choices=[], value=None),  # options_radio
             "",  # goal_box
@@ -561,7 +611,7 @@ def _load_status_task(uid, status):
 
     demo_video_path = None
     should_show = should_show_demo_video(actual_env_id) if actual_env_id else False
-    initial_log_msg = format_log_markdown(_ui_text("log", "action_selection_prompt"))
+    initial_log_msg = _action_selection_log()
 
     if should_show:
         if session.demonstration_frames:
@@ -594,7 +644,7 @@ def _load_status_task(uid, status):
         return (
             uid,
             gr.update(visible=True),  # main_interface
-            gr.update(value=img, interactive=False),  # img_display
+            _live_obs_update(value=img, interactive=False),  # img_display
             initial_log_msg,  # log_output
             gr.update(choices=radio_choices, value=None),  # options_radio
             goal_text,  # goal_box
@@ -619,7 +669,7 @@ def _load_status_task(uid, status):
     return (
         uid,
         gr.update(visible=True),  # main_interface
-        gr.update(value=img, interactive=False),  # img_display
+        _live_obs_update(value=img, interactive=False),  # img_display
         initial_log_msg,  # log_output
         gr.update(choices=radio_choices, value=None),  # options_radio
         goal_text,  # goal_box
@@ -743,22 +793,14 @@ def on_map_click(uid, option_value, evt: gr.SelectData):
     session = get_session(uid)
     if not session:
         LOGGER.warning("on_map_click: missing session uid=%s", _uid_for_log(uid))
-        return None, _ui_text("log", "session_error")
+        return (
+            _live_obs_update(value=None, interactive=False),
+            _ui_text("coords", "not_needed"),
+            format_log_markdown(_ui_text("log", "session_error")),
+        )
         
     # Check if current option actually needs coordinates
-    needs_coords = False
-    if option_value is not None:
-        # Parse option index similar to on_option_select
-        option_idx = None
-        if isinstance(option_value, tuple):
-             _, option_idx = option_value
-        else:
-             option_idx = option_value
-             
-        if option_idx is not None and 0 <= option_idx < len(session.raw_solve_options):
-             opt = session.raw_solve_options[option_idx]
-             if opt.get("available"):
-                 needs_coords = True
+    needs_coords = _option_requires_coords(session, option_value)
     
     if not needs_coords:
         LOGGER.debug(
@@ -770,7 +812,7 @@ def on_map_click(uid, option_value, evt: gr.SelectData):
         # Return current state without changes (or reset to default message if needed, but it should already be there)
         # We return the clean image and the "No need" message to enforce state
         base_img = session.get_pil_image(use_segmented=USE_SEGMENTED_VIEW)
-        return base_img, _ui_text("coords", "not_needed")
+        return _live_obs_update(value=base_img, interactive=False), _ui_text("coords", "not_needed"), _action_selection_log()
 
     x, y = evt.index[0], evt.index[1]
     LOGGER.debug(
@@ -789,7 +831,7 @@ def on_map_click(uid, option_value, evt: gr.SelectData):
     
     coords_str = f"{x}, {y}"
     
-    return marked_img, coords_str
+    return _live_obs_update(value=marked_img, interactive=True), coords_str, _action_selection_log()
 
 
 def _is_valid_coords_text(coords_text: str) -> bool:
@@ -821,7 +863,9 @@ def on_option_select(uid, option_value, coords_str=None):
     
     if option_value is None:
         LOGGER.debug("on_option_select uid=%s option=None", _uid_for_log(uid))
-        return default_msg, gr.update(interactive=False)
+        session = get_session(uid) if uid else None
+        base_img = session.get_pil_image(use_segmented=USE_SEGMENTED_VIEW) if session else _LIVE_OBS_UPDATE_SKIP
+        return default_msg, _live_obs_update(value=base_img, interactive=False), _action_selection_log()
     
     # 更新session活动时间（选择选项操作）
     if uid:
@@ -830,30 +874,27 @@ def on_option_select(uid, option_value, coords_str=None):
     session = get_session(uid)
     if not session:
         LOGGER.warning("on_option_select: missing session uid=%s", _uid_for_log(uid))
-        return default_msg, gr.update(interactive=False)
+        return default_msg, _live_obs_update(interactive=False), format_log_markdown(_ui_text("log", "session_error"))
     
-    # option_value 是 (label, idx) 元组或直接是 idx
-    if isinstance(option_value, tuple):
-        _, option_idx = option_value
-    else:
-        option_idx = option_value
+    option_idx = _parse_option_idx(option_value)
+    base_img = session.get_pil_image(use_segmented=USE_SEGMENTED_VIEW)
 
     # Determine coords message
-    if 0 <= option_idx < len(session.raw_solve_options):
-        opt = session.raw_solve_options[option_idx]
-        if opt.get("available"):
-             LOGGER.debug(
-                 "on_option_select uid=%s option=%s requires_coords=True valid_coords=%s",
-                 _uid_for_log(uid),
-                 option_idx,
-                 _is_valid_coords_text(coords_str),
-             )
-             if _is_valid_coords_text(coords_str):
-                 return coords_str, gr.update(interactive=True)
-             return _ui_text("coords", "select_keypoint"), gr.update(interactive=True)
+    if _option_requires_coords(session, option_idx):
+        LOGGER.debug(
+            "on_option_select uid=%s option=%s requires_coords=True valid_coords=%s",
+            _uid_for_log(uid),
+            option_idx,
+            _is_valid_coords_text(coords_str),
+        )
+        return (
+            _ui_text("coords", "select_keypoint"),
+            _live_obs_update(value=base_img, interactive=True, waiting_for_keypoint=True),
+            _keypoint_selection_log(),
+        )
     
     LOGGER.debug("on_option_select uid=%s option=%s requires_coords=False", _uid_for_log(uid), option_idx)
-    return default_msg, gr.update(interactive=False)
+    return default_msg, _live_obs_update(value=base_img, interactive=False), _action_selection_log()
 
 
 def on_reference_action(uid):
@@ -867,7 +908,7 @@ def on_reference_action(uid):
     if not session:
         LOGGER.warning("on_reference_action: missing session uid=%s", _uid_for_log(uid))
         return (
-            None,
+            _live_obs_update(value=None, interactive=False),
             gr.update(),
             _ui_text("coords", "not_needed"),
             format_log_markdown(_ui_text("log", "session_error")),
@@ -881,7 +922,7 @@ def on_reference_action(uid):
     except Exception as exc:
         LOGGER.exception("on_reference_action failed uid=%s", _uid_for_log(uid))
         return (
-            current_img,
+            _live_obs_update(value=current_img, interactive=False),
             gr.update(),
             gr.update(),
             format_log_markdown(_ui_text("log", "reference_action_error", error=exc)),
@@ -892,7 +933,7 @@ def on_reference_action(uid):
         if isinstance(reference, dict) and reference.get("message"):
             message = str(reference.get("message"))
         return (
-            current_img,
+            _live_obs_update(value=current_img, interactive=False),
             gr.update(),
             gr.update(),
             format_log_markdown(_ui_text("log", "reference_action_status", message=message)),
@@ -935,7 +976,7 @@ def on_reference_action(uid):
     )
 
     return (
-        updated_img,
+        _live_obs_update(value=updated_img, interactive=False),
         gr.update(value=option_idx),
         coords_text,
         format_log_markdown(log_text),
@@ -979,21 +1020,13 @@ def precheck_execute_inputs(uid, option_idx, coords_str):
         LOGGER.error("precheck_execute_inputs: missing session uid=%s", _uid_for_log(uid))
         raise gr.Error(_ui_text("log", "session_error"))
 
-    parsed_option_idx = option_idx
-    if isinstance(option_idx, tuple):
-        _, parsed_option_idx = option_idx
+    parsed_option_idx = _parse_option_idx(option_idx)
 
     if parsed_option_idx is None:
         LOGGER.debug("precheck_execute_inputs uid=%s missing option", _uid_for_log(uid))
         raise gr.Error(_ui_text("log", "execute_missing_action"))
 
-    needs_coords = False
-    if (
-        isinstance(parsed_option_idx, int)
-        and 0 <= parsed_option_idx < len(session.raw_solve_options)
-    ):
-        opt = session.raw_solve_options[parsed_option_idx]
-        needs_coords = bool(opt.get("available"))
+    needs_coords = _option_requires_coords(session, parsed_option_idx)
 
     if needs_coords and not _is_valid_coords_text(coords_str):
         LOGGER.debug(
@@ -1038,7 +1071,7 @@ def execute_step(uid, option_idx, coords_str):
     if not session:
         LOGGER.error("execute_step missing session uid=%s", _uid_for_log(uid))
         return (
-            None,
+            _live_obs_update(value=None, interactive=False),
             format_log_markdown(_ui_text("log", "session_error")),
             gr.update(),
             gr.update(),
@@ -1073,10 +1106,11 @@ def execute_step(uid, option_idx, coords_str):
         LOGGER.debug("execute_step uid=%s base_frames empty; triggering update_observation", _uid_for_log(uid))
         session.update_observation(use_segmentation=USE_SEGMENTED_VIEW)
     
+    option_idx = _parse_option_idx(option_idx)
     if option_idx is None:
         LOGGER.debug("execute_step uid=%s aborted: option_idx is None", _uid_for_log(uid))
         return (
-            session.get_pil_image(use_segmented=USE_SEGMENTED_VIEW),
+            _live_obs_update(value=session.get_pil_image(use_segmented=USE_SEGMENTED_VIEW), interactive=False),
             format_log_markdown(_ui_text("log", "execute_missing_action")),
             gr.update(),
             gr.update(),
@@ -1084,12 +1118,7 @@ def execute_step(uid, option_idx, coords_str):
             gr.update(interactive=True),
         )
 
-    # 检查当前选项是否需要坐标
-    needs_coords = False
-    if option_idx is not None and 0 <= option_idx < len(session.raw_solve_options):
-        opt = session.raw_solve_options[option_idx]
-        if opt.get("available"):
-            needs_coords = True
+    needs_coords = _option_requires_coords(session, option_idx)
     
     # 如果选项需要坐标，检查是否已经点击了图片
     if needs_coords:
@@ -1102,7 +1131,7 @@ def execute_step(uid, option_idx, coords_str):
             )
             current_img = session.get_pil_image(use_segmented=USE_SEGMENTED_VIEW)
             error_msg = _ui_text("coords", "select_keypoint_before_execute")
-            return current_img, format_log_markdown(error_msg), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
+            return _live_obs_update(value=current_img, interactive=False), format_log_markdown(error_msg), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
 
     # Parse coords
     click_coords = None
@@ -1257,7 +1286,7 @@ def execute_step(uid, option_idx, coords_str):
     )
     
     return (
-        img,
+        _live_obs_update(value=img, interactive=False),
         formatted_status,
         task_update,
         progress_update,

gradio-web/main.py

@@ -17,15 +17,9 @@ TEMP_DEMOS_DIR = PROJECT_ROOT / "temp_demos"
 CWD_TEMP_DEMOS_DIR = Path.cwd() / "temp_demos"
 
 
-def configure_runtime_devices():
-    """Restrict the app to physical GPU 1 and map rendering to the visible device."""
-    os.environ["CUDA_VISIBLE_DEVICES"] = "1"
-    os.environ.setdefault("NVIDIA_VISIBLE_DEVICES", "1")
-    # After masking to physical GPU 1, libraries should use logical cuda:0.
-    os.environ["SAPIEN_RENDER_DEVICE"] = "cuda:0"
 
 
-configure_runtime_devices()
+
 
 if str(PROJECT_ROOT) not in sys.path:
     sys.path.insert(0, str(PROJECT_ROOT))
@@ -132,7 +126,8 @@ def main():
 
     os.environ.setdefault("ROBOMME_TEMP_DEMOS_DIR", str(TEMP_DEMOS_DIR))
     allowed_paths = build_allowed_paths()
-    server_port = int(os.getenv("PORT", "7860"))
+    #server_port = int(os.getenv("PORT", "7860"))
+    server_port = 7861
     LOGGER.info(
         "Launching UI with server_name=%s server_port=%s ROBOMME_TEMP_DEMOS_DIR=%s",
         "0.0.0.0",

gradio-web/test/test_live_obs_refresh.py

@@ -21,6 +21,7 @@ def test_refresh_live_obs_skips_when_not_execution_phase(monkeypatch, reload_mod
 
 
 def test_refresh_live_obs_updates_image_from_latest_frame(monkeypatch, reload_module):
+    config = reload_module("config")
     callbacks = reload_module("gradio_callbacks")
     frame0 = np.zeros((8, 8, 3), dtype=np.uint8)
     frame1 = np.full((8, 8, 3), 11, dtype=np.uint8)
@@ -29,6 +30,7 @@ def test_refresh_live_obs_updates_image_from_latest_frame(monkeypatch, reload_mo
     frame4 = np.full((8, 8, 3), 44, dtype=np.uint8)
     session = _FakeSession([frame0])
     monkeypatch.setattr(callbacks, "get_session", lambda uid: session)
+    monkeypatch.setattr(callbacks, "KEYFRAME_DOWNSAMPLE_FACTOR", 2)
 
     # Reset queue state at execute start (cursor anchored at current base_frames length).
     callbacks.switch_to_execute_phase("uid-2")
@@ -41,11 +43,13 @@ def test_refresh_live_obs_updates_image_from_latest_frame(monkeypatch, reload_mo
 
     assert update1.get("__type__") == "update"
     assert update1.get("interactive") is False
+    assert update1.get("elem_classes") == config.get_live_obs_elem_classes()
     assert isinstance(update1.get("value"), Image.Image)
     assert update1["value"].getpixel((0, 0)) == (11, 11, 11)
 
     assert update2.get("__type__") == "update"
     assert update2.get("interactive") is False
+    assert update2.get("elem_classes") == config.get_live_obs_elem_classes()
     assert isinstance(update2.get("value"), Image.Image)
     assert update2["value"].getpixel((0, 0)) == (33, 33, 33)
 
@@ -55,16 +59,19 @@ def test_refresh_live_obs_updates_image_from_latest_frame(monkeypatch, reload_mo
 
 
 def test_switch_phase_keeps_live_obs_visible_and_toggles_interactive(reload_module):
+    config = reload_module("config")
     callbacks = reload_module("gradio_callbacks")
 
     to_exec = callbacks.switch_to_execute_phase("uid-3")
     assert len(to_exec) == 6
     assert to_exec[0].get("interactive") is False
     assert to_exec[4].get("interactive") is False
+    assert to_exec[4].get("elem_classes") == config.get_live_obs_elem_classes()
     assert to_exec[5].get("interactive") is False
 
     to_action = callbacks.switch_to_action_phase()
     assert len(to_action) == 6
     assert to_action[0].get("interactive") is True
     assert to_action[4].get("interactive") is True
+    assert to_action[4].get("elem_classes") == config.get_live_obs_elem_classes()
     assert to_action[5].get("interactive") is True

gradio-web/test/test_reference_action_callbacks.py

@@ -20,6 +20,10 @@ class _FakeOptionSession:
         self.raw_solve_options = [{"available": [object()]}]
         self.available_options = [("pick", 0)]
 
+    def get_pil_image(self, use_segmented=True):
+        _ = use_segmented
+        return Image.new("RGB", (24, 24), color=(0, 0, 0))
+
 
 def test_on_reference_action_success_updates_option_and_coords(monkeypatch, reload_module):
     config = reload_module("config")
@@ -40,10 +44,12 @@ def test_on_reference_action_success_updates_option_and_coords(monkeypatch, relo
     monkeypatch.setattr(callbacks, "update_session_activity", lambda uid: None)
     monkeypatch.setattr(callbacks, "get_session", lambda uid: session)
 
-    img, option_update, coords_text, log_html = callbacks.on_reference_action("uid-1")
+    img_update, option_update, coords_text, log_html = callbacks.on_reference_action("uid-1")
 
-    assert isinstance(img, Image.Image)
-    assert img.getpixel((5, 6)) != (0, 0, 0)
+    assert img_update.get("__type__") == "update"
+    assert isinstance(img_update.get("value"), Image.Image)
+    assert img_update["value"].getpixel((5, 6)) != (0, 0, 0)
+    assert img_update.get("elem_classes") == config.get_live_obs_elem_classes()
     assert option_update.get("value") == 2
     assert coords_text == "5, 6"
     expected_log = config.UI_TEXT["log"]["reference_action_message_with_coords"].format(
@@ -61,9 +67,10 @@ def test_on_reference_action_session_missing(monkeypatch, reload_module):
     monkeypatch.setattr(callbacks, "update_session_activity", lambda uid: None)
     monkeypatch.setattr(callbacks, "get_session", lambda uid: None)
 
-    img, option_update, coords_text, log_html = callbacks.on_reference_action("uid-missing")
+    img_update, option_update, coords_text, log_html = callbacks.on_reference_action("uid-missing")
 
-    assert img is None
+    assert img_update.get("__type__") == "update"
+    assert img_update.get("value") is None
     assert option_update.get("__type__") == "update"
     assert coords_text == config.UI_TEXT["coords"]["not_needed"]
     assert log_html == config.UI_TEXT["log"]["session_error"]
@@ -81,18 +88,39 @@ def test_on_reference_action_error_message_from_reference(monkeypatch, reload_mo
     assert log_html == config.UI_TEXT["log"]["reference_action_status"].format(message="bad ref")
 
 
-def test_on_option_select_keeps_valid_coords_when_option_needs_coords(monkeypatch, reload_module):
-    reload_module("config")
+def test_on_option_select_resets_to_keypoint_wait_state_for_point_action(monkeypatch, reload_module):
+    config = reload_module("config")
     callbacks = reload_module("gradio_callbacks")
 
     session = _FakeOptionSession()
     monkeypatch.setattr(callbacks, "update_session_activity", lambda uid: None)
     monkeypatch.setattr(callbacks, "get_session", lambda uid: session)
 
-    coords_text, img_update = callbacks.on_option_select("uid-1", 0, "12, 34")
+    coords_text, img_update, log_text = callbacks.on_option_select("uid-1", 0, "12, 34")
 
-    assert coords_text == "12, 34"
+    assert coords_text == config.UI_TEXT["coords"]["select_keypoint"]
     assert img_update.get("interactive") is True
+    assert img_update.get("elem_classes") == config.get_live_obs_elem_classes(waiting_for_keypoint=True)
+    assert log_text == config.UI_TEXT["log"]["keypoint_selection_prompt"]
+
+
+def test_on_map_click_clears_wait_state_and_restores_action_prompt(monkeypatch, reload_module):
+    config = reload_module("config")
+    callbacks = reload_module("gradio_callbacks")
+
+    session = _FakeOptionSession()
+    event = type("Evt", (), {"index": (5, 6)})()
+    monkeypatch.setattr(callbacks, "update_session_activity", lambda uid: None)
+    monkeypatch.setattr(callbacks, "get_session", lambda uid: session)
+
+    img_update, coords_text, log_text = callbacks.on_map_click("uid-1", 0, event)
+
+    assert img_update.get("__type__") == "update"
+    assert isinstance(img_update.get("value"), Image.Image)
+    assert img_update["value"].getpixel((5, 6)) != (0, 0, 0)
+    assert img_update.get("elem_classes") == config.get_live_obs_elem_classes()
+    assert coords_text == "5, 6"
+    assert log_text == config.UI_TEXT["log"]["action_selection_prompt"]
 
 
 def test_on_reference_action_uses_configured_action_text_override(monkeypatch, reload_module):
@@ -120,5 +148,5 @@ def test_on_reference_action_uses_configured_action_text_override(monkeypatch, r
     assert coords_text == config.UI_TEXT["coords"]["not_needed"]
     assert log_html == config.UI_TEXT["log"]["reference_action_message"].format(
         option_label="a",
-        option_action="move forward↑",
+        option_action="move forward↓",
     )

gradio-web/test/test_ui_native_layout_contract.py

@@ -45,6 +45,20 @@ def test_native_ui_css_excludes_header_title_from_global_font_size(reload_module
     assert "font-size: var(--text-xxl) !important;" in ui_layout.CSS
 
 
+def test_native_ui_css_highlights_media_card_not_live_obs_transform(reload_module):
+    ui_layout = reload_module("ui_layout")
+
+    css = ui_layout.CSS
+
+    assert "#media_card::after" in css
+    assert "#media_card:has(#live_obs.live-obs-keypoint-waiting)::after" in css
+    assert "animation: media-card-keypoint-ring 1.2s ease-in-out infinite;" in css
+    assert "@keyframes media-card-keypoint-ring" in css
+    assert "#live_obs.live-obs-keypoint-waiting .image-frame" not in css
+    assert "#live_obs.live-obs-keypoint-waiting .upload-container" not in css
+    assert "transform: scale(" not in css
+
+
 def test_extract_last_goal_prefers_last_list_item(reload_module):
     ui_layout = reload_module("ui_layout")
 

gradio-web/test/test_ui_phase_machine_runtime_e2e.py

@@ -100,6 +100,68 @@ def _read_coords_box_value(page) -> str | None:
     )
 
 
+def _read_log_output_value(page) -> str | None:
+    return page.evaluate(
+        """() => {
+            const root = document.getElementById('log_output');
+            if (!root) return null;
+            const field = root.querySelector('textarea, input');
+            if (field && typeof field.value === 'string') {
+                const value = field.value.trim();
+                return value || null;
+            }
+            const value = (root.textContent || '').trim();
+            return value || null;
+        }"""
+    )
+
+
+def _read_elem_classes(page, elem_id: str) -> list[str] | None:
+    return page.evaluate(
+        """(elemId) => {
+            const root = document.getElementById(elemId);
+            return root ? Array.from(root.classList) : null;
+        }""",
+        elem_id,
+    )
+
+
+def _read_media_card_wait_snapshot(page) -> dict[str, str | float | None]:
+    return page.evaluate(
+        """() => {
+            const card = document.getElementById('media_card');
+            if (!card) {
+                return {
+                    opacity: null,
+                    borderColor: null,
+                    boxShadow: null,
+                    animationName: null,
+                };
+            }
+            const style = getComputedStyle(card, '::after');
+            return {
+                opacity: Number.parseFloat(style.opacity || '0'),
+                borderColor: style.borderColor || null,
+                boxShadow: style.boxShadow || null,
+                animationName: style.animationName || null,
+            };
+        }"""
+    )
+
+
+def _read_live_obs_transform_snapshot(page) -> dict[str, str | None]:
+    return page.evaluate(
+        """() => {
+            const img = document.querySelector('#live_obs img');
+            const frame = document.querySelector('#live_obs .image-frame');
+            return {
+                imgTransform: img ? getComputedStyle(img).transform : null,
+                frameTransform: frame ? getComputedStyle(frame).transform : null,
+            };
+        }"""
+    )
+
+
 def _read_phase_visibility(page) -> dict[str, bool | str | None]:
     return page.evaluate(
         """() => {
@@ -986,6 +1048,207 @@ def test_no_video_task_hides_manual_demo_button(monkeypatch):
         demo.close()
 
 
+def test_keypoint_wait_state_pulses_live_obs_and_updates_system_log(monkeypatch):
+    config_module = importlib.reload(importlib.import_module("config"))
+    callbacks = importlib.reload(importlib.import_module("gradio_callbacks"))
+    ui_layout = importlib.reload(importlib.import_module("ui_layout"))
+
+    fake_obs = np.zeros((24, 48, 3), dtype=np.uint8)
+    fake_obs[:, :] = [15, 20, 25]
+    fake_obs_img = Image.fromarray(fake_obs)
+
+    class FakeSession:
+        raw_solve_options = [{"available": [object()]}, {"available": False}]
+
+        def get_pil_image(self, use_segmented=False):
+            _ = use_segmented
+            return fake_obs_img.copy()
+
+    def fake_init_app(_request=None):
+        return (
+            "uid-keypoint-wait",
+            gr.update(visible=True),  # main_interface
+            gr.update(
+                value=fake_obs_img.copy(),
+                interactive=False,
+                elem_classes=config_module.get_live_obs_elem_classes(),
+            ),  # img_display
+            config_module.UI_TEXT["log"]["action_selection_prompt"],  # log_output
+            gr.update(choices=[("pick", 0), ("skip", 1)], value=None),  # options_radio
+            "goal",  # goal_box
+            gr.update(
+                value=config_module.UI_TEXT["coords"]["not_needed"],
+                visible=True,
+                interactive=False,
+            ),  # coords_box
+            gr.update(value=None, visible=False),  # video_display
+            gr.update(visible=False, interactive=False),  # watch_demo_video_btn
+            "KeypointEnv (Episode 1)",  # task_info_box
+            "Completed: 0",  # progress_info_box
+            gr.update(interactive=True),  # restart_episode_btn
+            gr.update(interactive=True),  # next_task_btn
+            gr.update(interactive=True),  # exec_btn
+            gr.update(visible=False),  # video_phase_group
+            gr.update(visible=True),  # action_phase_group
+            gr.update(visible=True),  # control_panel_group
+            gr.update(value="hint"),  # task_hint_display
+            gr.update(visible=False),  # loading_overlay
+            gr.update(interactive=True),  # reference_action_btn
+        )
+
+    monkeypatch.setattr(ui_layout, "init_app", fake_init_app)
+    monkeypatch.setattr(callbacks, "get_session", lambda uid: FakeSession())
+    monkeypatch.setattr(callbacks, "update_session_activity", lambda uid: None)
+
+    demo = ui_layout.create_ui_blocks()
+
+    port = _free_port()
+    host = "127.0.0.1"
+    root_url = f"http://{host}:{port}/"
+
+    app = FastAPI(title="keypoint-wait-state-test")
+    app = gr.mount_gradio_app(app, demo, path="/")
+
+    config = uvicorn.Config(app, host=host, port=port, log_level="error")
+    server = uvicorn.Server(config)
+    thread = threading.Thread(target=server.run, daemon=True)
+    thread.start()
+    _wait_http_ready(root_url)
+
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page(viewport={"width": 1280, "height": 900})
+            page.goto(root_url, wait_until="domcontentloaded")
+            page.add_style_tag(content=ui_layout.CSS)
+            page.wait_for_selector("#main_interface_root", state="visible", timeout=15000)
+            page.wait_for_selector("#live_obs img", timeout=15000)
+
+            initial_classes = _read_elem_classes(page, "live_obs")
+            assert initial_classes is not None
+            assert config_module.LIVE_OBS_KEYPOINT_WAIT_CLASS not in initial_classes
+            assert _read_log_output_value(page) == config_module.UI_TEXT["log"]["action_selection_prompt"]
+            initial_card_wait = _read_media_card_wait_snapshot(page)
+            initial_transforms = _read_live_obs_transform_snapshot(page)
+            initial_img_box = page.locator("#live_obs img").bounding_box()
+            initial_frame_box = page.locator("#live_obs .image-frame").bounding_box()
+            assert initial_card_wait["opacity"] == 0
+            assert initial_card_wait["animationName"] == "none"
+            assert initial_transforms["imgTransform"] == "none"
+            assert initial_transforms["frameTransform"] == "none"
+            assert initial_img_box is not None
+            assert initial_frame_box is not None
+
+            page.locator("#action_radio input[type='radio']").first.check(force=True)
+
+            page.wait_for_function(
+                """(state) => {
+                    const liveObs = document.getElementById('live_obs');
+                    const coordsRoot = document.getElementById('coords_box');
+                    const coordsField = coordsRoot?.querySelector('textarea, input');
+                    const logRoot = document.getElementById('log_output');
+                    const logField = logRoot?.querySelector('textarea, input');
+                    const mediaCard = document.getElementById('media_card');
+                    const mediaAfter = mediaCard ? getComputedStyle(mediaCard, '::after') : null;
+                    const coordsValue = coordsField ? coordsField.value.trim() : '';
+                    const logValue = logField ? logField.value.trim() : (logRoot?.textContent || '').trim();
+                    return (
+                        !!liveObs &&
+                        liveObs.classList.contains(state.waitClass) &&
+                        !!mediaAfter &&
+                        Number.parseFloat(mediaAfter.opacity || '0') > 0.5 &&
+                        mediaAfter.animationName === state.cardAnimation &&
+                        coordsValue === state.coordsPrompt &&
+                        logValue === state.waitLog
+                    );
+                }""",
+                arg={
+                    "cardAnimation": "media-card-keypoint-ring",
+                    "waitClass": config_module.LIVE_OBS_KEYPOINT_WAIT_CLASS,
+                    "coordsPrompt": config_module.UI_TEXT["coords"]["select_keypoint"],
+                    "waitLog": config_module.UI_TEXT["log"]["keypoint_selection_prompt"],
+                },
+                timeout=5000,
+            )
+
+            wait_classes = _read_elem_classes(page, "live_obs")
+            assert wait_classes is not None
+            assert config_module.LIVE_OBS_KEYPOINT_WAIT_CLASS in wait_classes
+            assert _read_coords_box_value(page) == config_module.UI_TEXT["coords"]["select_keypoint"]
+            assert _read_log_output_value(page) == config_module.UI_TEXT["log"]["keypoint_selection_prompt"]
+            wait_card = _read_media_card_wait_snapshot(page)
+            wait_transforms = _read_live_obs_transform_snapshot(page)
+            wait_img_box = page.locator("#live_obs img").bounding_box()
+            wait_frame_box = page.locator("#live_obs .image-frame").bounding_box()
+            assert wait_card["opacity"] is not None and wait_card["opacity"] > 0.5
+            assert wait_card["animationName"] == "media-card-keypoint-ring"
+            assert wait_card["borderColor"] != "rgba(225, 29, 72, 0)"
+            assert wait_transforms["imgTransform"] == "none"
+            assert wait_transforms["frameTransform"] == "none"
+            assert wait_img_box is not None
+            assert wait_frame_box is not None
+            assert wait_img_box["x"] == pytest.approx(initial_img_box["x"], abs=1.0)
+            assert wait_img_box["y"] == pytest.approx(initial_img_box["y"], abs=1.0)
+            assert wait_img_box["width"] == pytest.approx(initial_img_box["width"], abs=1.0)
+            assert wait_img_box["height"] == pytest.approx(initial_img_box["height"], abs=1.0)
+            assert wait_frame_box["x"] == pytest.approx(initial_frame_box["x"], abs=1.0)
+            assert wait_frame_box["y"] == pytest.approx(initial_frame_box["y"], abs=1.0)
+            assert wait_frame_box["width"] == pytest.approx(initial_frame_box["width"], abs=1.0)
+            assert wait_frame_box["height"] == pytest.approx(initial_frame_box["height"], abs=1.0)
+
+            box = page.locator("#live_obs img").bounding_box()
+            assert box is not None
+            target_x = box["x"] + ((24.5) / 48.0) * box["width"]
+            target_y = box["y"] + ((8.5) / 24.0) * box["height"]
+            page.mouse.click(target_x, target_y)
+
+            page.wait_for_function(
+                """(state) => {
+                    const liveObs = document.getElementById('live_obs');
+                    const coordsRoot = document.getElementById('coords_box');
+                    const coordsField = coordsRoot?.querySelector('textarea, input');
+                    const logRoot = document.getElementById('log_output');
+                    const logField = logRoot?.querySelector('textarea, input');
+                    const coordsValue = coordsField ? coordsField.value.trim() : '';
+                    const logValue = logField ? logField.value.trim() : (logRoot?.textContent || '').trim();
+                    return (
+                        !!liveObs &&
+                        !liveObs.classList.contains(state.waitClass) &&
+                        /^\\d+\\s*,\\s*\\d+$/.test(coordsValue) &&
+                        logValue === state.actionLog
+                    );
+                }""",
+                arg={
+                    "waitClass": config_module.LIVE_OBS_KEYPOINT_WAIT_CLASS,
+                    "actionLog": config_module.UI_TEXT["log"]["action_selection_prompt"],
+                },
+                timeout=5000,
+            )
+
+            coords_value = _read_coords_box_value(page)
+            assert coords_value is not None
+            coord_x, coord_y = [int(part.strip()) for part in coords_value.split(",", 1)]
+            assert abs(coord_x - 24) <= 1
+            assert abs(coord_y - 8) <= 1
+            final_classes = _read_elem_classes(page, "live_obs")
+            assert final_classes is not None
+            assert config_module.LIVE_OBS_KEYPOINT_WAIT_CLASS not in final_classes
+            assert config_module.LIVE_OBS_BASE_CLASS in final_classes
+            assert _read_log_output_value(page) == config_module.UI_TEXT["log"]["action_selection_prompt"]
+            final_card_wait = _read_media_card_wait_snapshot(page)
+            final_transforms = _read_live_obs_transform_snapshot(page)
+            assert final_card_wait["opacity"] == 0
+            assert final_card_wait["animationName"] == "none"
+            assert final_transforms["imgTransform"] == "none"
+            assert final_transforms["frameTransform"] == "none"
+
+            browser.close()
+    finally:
+        server.should_exit = True
+        thread.join(timeout=10)
+        demo.close()
+
+
 def test_live_obs_client_resize_fills_width_and_keeps_click_mapping(monkeypatch):
     callbacks = importlib.reload(importlib.import_module("gradio_callbacks"))
     ui_layout = importlib.reload(importlib.import_module("ui_layout"))

gradio-web/test/test_ui_text_config.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import pytest
+from PIL import Image
 
 
 class _FakeOptionSession:
@@ -8,6 +9,10 @@ class _FakeOptionSession:
         self.env_id = env_id
         self.raw_solve_options = raw_solve_options or [{"available": True}]
 
+    def get_pil_image(self, use_segmented=False):
+        _ = use_segmented
+        return Image.new("RGB", (8, 8), color=(0, 0, 0))
+
 
 class _FakeLoadSession:
     def __init__(self, env_id, available_options, raw_solve_options, demonstration_frames=None, language_goal=""):
@@ -25,18 +30,25 @@ class _FakeLoadSession:
         return "IMG"
 
 
-def test_on_option_select_uses_configured_select_keypoint_message(monkeypatch, reload_module):
+def test_on_option_select_uses_configured_select_keypoint_and_log_messages(monkeypatch, reload_module):
     reload_module("config")
     callbacks = reload_module("gradio_callbacks")
 
     monkeypatch.setitem(callbacks.UI_TEXT["coords"], "select_keypoint", "pick a point from config")
+    monkeypatch.setitem(
+        callbacks.UI_TEXT["log"],
+        "keypoint_selection_prompt",
+        "custom log prompt from config",
+    )
     monkeypatch.setattr(callbacks, "update_session_activity", lambda uid: None)
     monkeypatch.setattr(callbacks, "get_session", lambda uid: _FakeOptionSession())
 
-    coords_text, img_update = callbacks.on_option_select("uid-1", 0, None)
+    coords_text, img_update, log_text = callbacks.on_option_select("uid-1", 0, None)
 
     assert coords_text == "pick a point from config"
     assert img_update.get("interactive") is True
+    assert callbacks.get_live_obs_elem_classes(waiting_for_keypoint=True) == img_update.get("elem_classes")
+    assert log_text == "custom log prompt from config"
 
 
 def test_precheck_execute_inputs_uses_configured_before_execute_message(monkeypatch, reload_module):
@@ -101,12 +113,14 @@ def test_missing_session_paths_use_configured_session_error(monkeypatch, reload_
     monkeypatch.setattr(callbacks, "get_session", lambda uid: None)
 
     _img, _option_update, coords_text, log_text = callbacks.on_reference_action("uid-missing")
-    map_img, map_text = callbacks.on_map_click("uid-missing", None, None)
+    map_img, map_coords, map_log = callbacks.on_map_click("uid-missing", None, None)
 
     assert coords_text == callbacks.UI_TEXT["coords"]["not_needed"]
     assert log_text == "Session Error From Config"
-    assert map_img is None
-    assert map_text == "Session Error From Config"
+    assert map_img.get("__type__") == "update"
+    assert map_img.get("value") is None
+    assert map_coords == callbacks.UI_TEXT["coords"]["not_needed"]
+    assert map_log == "Session Error From Config"
 
 
 def test_get_ui_action_text_uses_configured_overrides_and_fallback(reload_module):

gradio-web/ui_layout.py

@@ -10,11 +10,14 @@ import gradio as gr
 
 from config import (
     CONTROL_PANEL_SCALE,
+    LIVE_OBS_BASE_CLASS,
+    LIVE_OBS_KEYPOINT_WAIT_CLASS,
     LIVE_OBS_REFRESH_HZ,
     KEYPOINT_SELECTION_SCALE,
     RIGHT_TOP_ACTION_SCALE,
     RIGHT_TOP_LOG_SCALE,
     UI_GLOBAL_FONT_SIZE,
+    get_live_obs_elem_classes,
 )
 from gradio_callbacks import (
     execute_step,
@@ -299,6 +302,60 @@ button#reference_action_btn:not(:disabled):hover {{
 button#watch_demo_video_btn {{
     width: 100%;
 }}
+
+#media_card {{
+    position: relative;
+    border-radius: 24px;
+    overflow: visible;
+}}
+
+#media_card::after {{
+    content: "";
+    position: absolute;
+    inset: -8px;
+    border-radius: 30px;
+    border: 3px solid rgba(225, 29, 72, 0.00);
+    box-shadow: 0 0 0 0 rgba(225, 29, 72, 0.00);
+    opacity: 0;
+    pointer-events: none;
+    transition: opacity 180ms ease, border-color 180ms ease, box-shadow 180ms ease;
+    z-index: 0;
+}}
+
+@keyframes media-card-keypoint-ring {{
+    0% {{
+        box-shadow: 0 0 0 0 rgba(225, 29, 72, 0.00);
+        border-color: rgba(225, 29, 72, 0.72);
+        opacity: 0.72;
+    }}
+    18% {{
+        box-shadow: 0 0 0 4px rgba(225, 29, 72, 0.28);
+        border-color: rgba(225, 29, 72, 0.94);
+        opacity: 1;
+    }}
+    36% {{
+        box-shadow: 0 0 0 10px rgba(225, 29, 72, 0.12);
+        border-color: rgba(225, 29, 72, 0.88);
+        opacity: 0.94;
+    }}
+    62% {{
+        box-shadow: 0 0 0 2px rgba(225, 29, 72, 0.18);
+        border-color: rgba(225, 29, 72, 0.96);
+        opacity: 1;
+    }}
+    100% {{
+        box-shadow: 0 0 0 0 rgba(225, 29, 72, 0.00);
+        border-color: rgba(225, 29, 72, 0.72);
+        opacity: 0.72;
+    }}
+}}
+
+#media_card:has(#live_obs.{LIVE_OBS_KEYPOINT_WAIT_CLASS})::after {{
+    border-color: rgba(225, 29, 72, 0.94);
+    box-shadow: 0 0 0 0 rgba(225, 29, 72, 0.00);
+    opacity: 1;
+    animation: media-card-keypoint-ring 1.2s ease-in-out infinite;
+}}
 """
 
 
@@ -439,7 +496,7 @@ def create_ui_blocks():
                                 interactive=False,
                                 type="pil",
                                 elem_id="live_obs",
-                                elem_classes=["live-obs-resizable"],
+                                elem_classes=get_live_obs_elem_classes(),
                                 show_label=True,
                                 buttons=[],
                                 sources=[],
@@ -739,13 +796,13 @@ def create_ui_blocks():
         img_display.select(
             fn=on_map_click,
             inputs=[uid_state, options_radio],
-            outputs=[img_display, coords_box],
+            outputs=[img_display, coords_box, log_output],
         )
 
         options_radio.change(
             fn=on_option_select,
             inputs=[uid_state, options_radio, coords_box],
-            outputs=[coords_box, img_display],
+            outputs=[coords_box, img_display, log_output],
         )
 
         watch_demo_video_btn.click(