system log executing!

gradio-web/config.py

@@ -65,6 +65,7 @@ UI_TEXT = {
     "log": {
         "action_selection_prompt": "Please select the action.\nActions with 🎯 need to select a point on the image as input",
         "point_selection_prompt": "Current action needs location input, please click on the image to select key pixel",
+        "execute_action_prompt": "Executing Action {label}",
         "demo_video_prompt": 'Press "Watch Video Input 🎬" to watch a video\nNote: you can only watch the video once',
         "session_error": "Session expired. Please refresh the page and try again.",
         "reference_action_error": "Ground Truth Action Error: {error}",

gradio-web/gradio_callbacks.py

@@ -62,10 +62,41 @@ def _point_selection_log():
     return format_log_markdown(_ui_text("log", "point_selection_prompt"))
 
 
+def _get_raw_option_label(session, option_idx):
+    try:
+        option_index = int(option_idx)
+    except (TypeError, ValueError):
+        return None
+
+    raw_solve_options = getattr(session, "raw_solve_options", None)
+    if not isinstance(raw_solve_options, list):
+        return None
+    if not (0 <= option_index < len(raw_solve_options)):
+        return None
+
+    raw_option = raw_solve_options[option_index]
+    if not isinstance(raw_option, dict):
+        return None
+
+    label = str(raw_option.get("label", "")).strip()
+    return label or None
+
+
+def _execution_video_log(session, option_idx, fallback_status=None):
+    label = _get_raw_option_label(session, option_idx)
+    if label:
+        return format_log_markdown(_ui_text("log", "execute_action_prompt", label=label))
+    if fallback_status is None:
+        return None
+    return format_log_markdown(fallback_status)
+
+
 def _default_post_execute_log_state():
     return {
         "preserve_terminal_log": False,
         "terminal_log_value": None,
+        "preserve_execute_video_log": False,
+        "execute_video_log_value": None,
     }
 
 
@@ -78,9 +109,17 @@ def _normalize_post_execute_log_state(state):
             preserve_terminal_log = False
         else:
             terminal_log_value = str(terminal_log_value)
+        preserve_execute_video_log = bool(state.get("preserve_execute_video_log", False))
+        execute_video_log_value = state.get("execute_video_log_value")
+        if execute_video_log_value is None:
+            preserve_execute_video_log = False
+        else:
+            execute_video_log_value = str(execute_video_log_value)
         return {
             "preserve_terminal_log": preserve_terminal_log,
             "terminal_log_value": terminal_log_value,
+            "preserve_execute_video_log": preserve_execute_video_log,
+            "execute_video_log_value": execute_video_log_value,
         }
     return _default_post_execute_log_state()
 
@@ -487,9 +526,11 @@ def on_execute_video_end_transition(
     """Transition from execute video phase back to the action phase."""
     controls_state = _normalize_post_execute_controls_state(post_execute_controls_state)
     log_state = _normalize_post_execute_log_state(post_execute_log_state)
-    log_update = gr.update()
+    next_log_state = _default_post_execute_log_state()
+    log_update = gr.update(value=_action_selection_log())
     if log_state["preserve_terminal_log"]:
         log_update = gr.update(value=log_state["terminal_log_value"])
+        next_log_state = log_state
     LOGGER.debug(
         "on_execute_video_end_transition uid=%s controls_state=%s log_state=%s",
         _uid_for_log(uid),
@@ -508,6 +549,7 @@ def on_execute_video_end_transition(
         log_update,  # log_output
         gr.update(interactive=controls_state["reference_action_interactive"]),  # reference_action_btn
         gr.update(interactive=True),  # task_hint_display
+        next_log_state,  # post_execute_log_state
         "action_point",  # ui_phase_state
     )
 
@@ -928,6 +970,20 @@ def on_option_select(
             False,
             normalized_post_execute_log_state,
         )
+
+    if normalized_post_execute_log_state["preserve_execute_video_log"]:
+        LOGGER.debug(
+            "on_option_select preserving execution-video log uid=%s option=%s",
+            _uid_for_log(uid),
+            option_value,
+        )
+        return (
+            gr.update(),
+            gr.update(),
+            gr.update(value=normalized_post_execute_log_state["execute_video_log_value"]),
+            False,
+            normalized_post_execute_log_state,
+        )
     
     if option_value is None:
         LOGGER.debug("on_option_select uid=%s option=None", _uid_for_log(uid))
@@ -1445,10 +1501,20 @@ def execute_step(uid, option_idx, coords_str):
     
     # 格式化日志消息为 HTML 格式（支持颜色显示）
     formatted_status = format_log_markdown(status)
+    if show_execution_video and not done:
+        formatted_status = _execution_video_log(session, option_idx, fallback_status=status) or formatted_status
+        post_execute_log_state = {
+            "preserve_terminal_log": False,
+            "terminal_log_value": None,
+            "preserve_execute_video_log": True,
+            "execute_video_log_value": formatted_status,
+        }
     if done:
         post_execute_log_state = {
             "preserve_terminal_log": True,
             "terminal_log_value": formatted_status,
+            "preserve_execute_video_log": False,
+            "execute_video_log_value": None,
         }
     LOGGER.debug(
         "execute_step done uid=%s env=%s ep=%s done=%s post_execute_controls=%s post_execute_log=%s show_execution_video=%s",

gradio-web/test/test_live_obs_refresh.py

@@ -7,7 +7,7 @@ class _FakeSession:
     def __init__(self):
         self.env_id = "BinFill"
         self.episode_idx = 1
-        self.raw_solve_options = [{"available": False}]
+        self.raw_solve_options = [{"label": "a", "available": False}]
         self.available_options = [("pick", 0)]
         self.base_frames = []
         self.last_execution_frames = []
@@ -63,8 +63,19 @@ def test_execute_step_builds_video_from_last_execution_frames(monkeypatch, reloa
     assert result[11]["value"] is None
     assert result[11]["interactive"] is False
     assert result[14]["interactive"] is False
-    assert result[15] is True
-    assert result[16] == "execution_video"
+    expected_log = callbacks.UI_TEXT["log"]["execute_action_prompt"].format(label="a")
+    assert result[1] == expected_log
+    assert result[15] == {
+        "exec_btn_interactive": True,
+        "reference_action_interactive": True,
+    }
+    assert result[16] == {
+        "preserve_terminal_log": False,
+        "terminal_log_value": None,
+        "preserve_execute_video_log": True,
+        "execute_video_log_value": expected_log,
+    }
+    assert result[17] == "execution_video"
 
 
 def test_execute_step_falls_back_to_single_frame_clip_when_no_new_frames(monkeypatch, reload_module):
@@ -98,7 +109,7 @@ def test_execute_step_falls_back_to_single_frame_clip_when_no_new_frames(monkeyp
     assert int(captured["frames"][0][0, 0, 0]) == 33
     assert result[7]["visible"] is True
     assert result[10]["visible"] is True
-    assert result[16] == "execution_video"
+    assert result[17] == "execution_video"
 
 
 def test_switch_phase_toggles_live_obs_interactive_without_refresh_queue(reload_module):

gradio-web/test/test_ui_phase_machine_runtime_e2e.py

@@ -2576,7 +2576,10 @@ def _run_local_execute_video_transition_test(
             self.episode_idx = 1
             self.language_goal = "place cube on target"
             self.available_options = [("pick", 0), ("point", 1)]
-            self.raw_solve_options = [{"available": False}, {"available": [object()]}]
+            self.raw_solve_options = [
+                {"label": "a", "action": "pick", "available": False},
+                {"label": "b", "action": "point", "available": [object()]},
+            ]
             self.demonstration_frames = []
             self.last_execution_frames = []
             self.base_frames = [fake_obs.copy()]
@@ -2624,6 +2627,8 @@ def _run_local_execute_video_transition_test(
                 value={
                     "preserve_terminal_log": False,
                     "terminal_log_value": None,
+                    "preserve_execute_video_log": False,
+                    "execute_video_log_value": None,
                 }
             )
             suppress_state = gr.State(value=False)
@@ -2705,6 +2710,12 @@ def _run_local_execute_video_transition_test(
                 outputs=[coords_box, img_display, log_output, suppress_state, post_execute_log_state],
                 queue=False,
             )
+            img_display.select(
+                fn=cb.on_map_click,
+                inputs=[uid_state, options_radio],
+                outputs=[img_display, coords_box, log_output],
+                queue=False,
+            )
 
             execute_video_display.end(
                 fn=cb.on_execute_video_end_transition,
@@ -2721,6 +2732,7 @@ def _run_local_execute_video_transition_test(
                     log_output,
                     reference_action_btn,
                     task_hint_display,
+                    post_execute_log_state,
                     phase_state,
                 ],
                 queue=False,
@@ -2740,6 +2752,7 @@ def _run_local_execute_video_transition_test(
                     log_output,
                     reference_action_btn,
                     task_hint_display,
+                    post_execute_log_state,
                     phase_state,
                 ],
                 queue=False,
@@ -2764,7 +2777,39 @@ def _run_local_execute_video_transition_test(
                 page = browser.new_page(viewport={"width": 1280, "height": 900})
                 page.goto(root_url, wait_until="domcontentloaded")
                 page.wait_for_selector("#main_interface", state="visible", timeout=20000)
-                page.locator("#action_radio input[type='radio']").first.check(force=True)
+                page.locator("#action_radio input[type='radio']").nth(1).check(force=True)
+                page.wait_for_function(
+                    """(state) => {
+                        const coordsRoot = document.getElementById('coords_box');
+                        const coordsField = coordsRoot?.querySelector('textarea, input');
+                        const logRoot = document.getElementById('log_output');
+                        const logField = logRoot?.querySelector('textarea, input');
+                        const coordsValue = coordsField ? coordsField.value.trim() : '';
+                        const logValue = logField ? logField.value.trim() : (logRoot?.textContent || '').trim();
+                        return coordsValue === state.coordsPrompt && logValue === state.waitLog;
+                    }""",
+                    arg={
+                        "coordsPrompt": config_module.UI_TEXT["coords"]["select_point"],
+                        "waitLog": config_module.UI_TEXT["log"]["point_selection_prompt"],
+                    },
+                    timeout=5000,
+                )
+                box = page.locator("#live_obs img").bounding_box()
+                assert box is not None
+                page.mouse.click(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2)
+                page.wait_for_function(
+                    """(state) => {
+                        const coordsRoot = document.getElementById('coords_box');
+                        const coordsField = coordsRoot?.querySelector('textarea, input');
+                        const logRoot = document.getElementById('log_output');
+                        const logField = logRoot?.querySelector('textarea, input');
+                        const coordsValue = coordsField ? coordsField.value.trim() : '';
+                        const logValue = logField ? logField.value.trim() : (logRoot?.textContent || '').trim();
+                        return /^\\d+\\s*,\\s*\\d+$/.test(coordsValue) && logValue === state.actionLog;
+                    }""",
+                    arg={"actionLog": config_module.UI_TEXT["log"]["action_selection_prompt"]},
+                    timeout=5000,
+                )
                 page.locator("#exec_btn button, button#exec_btn").first.click()
                 page.wait_for_selector("#execute_video video", timeout=5000)
                 page.wait_for_function(
@@ -2788,6 +2833,10 @@ def _run_local_execute_video_transition_test(
                     }""",
                     timeout=10000,
                 )
+                if not done:
+                    execution_log = _read_log_output_value(page)
+                    assert execution_log == config_module.UI_TEXT["log"]["execute_action_prompt"].format(label="b")
+                    assert execution_log != config_module.UI_TEXT["log"]["point_selection_prompt"]
                 controls_after_execute = _read_demo_video_controls(page, elem_id="execute_video", button_elem_id=None)
                 assert controls_after_execute["autoplay"] is True
                 assert controls_after_execute["paused"] is False
@@ -2881,30 +2930,7 @@ def _run_local_execute_video_transition_test(
                         "execDisabled": False,
                         "refDisabled": False,
                     }
-                    page.locator("#action_radio input[type='radio']").nth(1).check(force=True)
-                    page.wait_for_function(
-                        """(state) => {
-                            const liveObs = document.getElementById('live_obs');
-                            const coordsRoot = document.getElementById('coords_box');
-                            const coordsField = coordsRoot?.querySelector('textarea, input');
-                            const logRoot = document.getElementById('log_output');
-                            const logField = logRoot?.querySelector('textarea, input');
-                            const coordsValue = coordsField ? coordsField.value.trim() : '';
-                            const logValue = logField ? logField.value.trim() : (logRoot?.textContent || '').trim();
-                            return (
-                                !!liveObs &&
-                                liveObs.classList.contains(state.waitClass) &&
-                                coordsValue === state.coordsPrompt &&
-                                logValue === state.waitLog
-                            );
-                        }""",
-                        arg={
-                            "waitClass": config_module.LIVE_OBS_POINT_WAIT_CLASS,
-                            "coordsPrompt": config_module.UI_TEXT["coords"]["select_point"],
-                            "waitLog": config_module.UI_TEXT["log"]["point_selection_prompt"],
-                        },
-                        timeout=5000,
-                    )
+                    assert _read_log_output_value(page) == config_module.UI_TEXT["log"]["action_selection_prompt"]
 
                 browser.close()
         finally:
@@ -3039,6 +3065,8 @@ def test_phase_machine_runtime_stopcube_remain_static_merges_short_tail(monkeypa
             value={
                 "preserve_terminal_log": False,
                 "terminal_log_value": None,
+                "preserve_execute_video_log": False,
+                "execute_video_log_value": None,
             }
         )
         suppress_state = gr.State(value=False)
@@ -3140,6 +3168,7 @@ def test_phase_machine_runtime_stopcube_remain_static_merges_short_tail(monkeypa
                 log_output,
                 reference_action_btn,
                 task_hint_display,
+                post_execute_log_state,
                 phase_state,
             ],
             queue=False,
@@ -3159,6 +3188,7 @@ def test_phase_machine_runtime_stopcube_remain_static_merges_short_tail(monkeypa
                 log_output,
                 reference_action_btn,
                 task_hint_display,
+                post_execute_log_state,
                 phase_state,
             ],
             queue=False,

gradio-web/test/test_ui_text_config.py

@@ -103,10 +103,33 @@ def test_on_execute_video_end_transition_restores_controls_for_non_terminal_stat
     assert result[4]["interactive"] is True
     assert result[5]["interactive"] is True
     assert result[6]["interactive"] is True
-    assert result[8].get("__type__") == "update"
+    assert result[8]["value"] == callbacks.UI_TEXT["log"]["action_selection_prompt"]
     assert result[9]["interactive"] is True
     assert result[10]["interactive"] is True
-    assert result[11] == "action_point"
+    assert result[11] == callbacks._default_post_execute_log_state()
+    assert result[12] == "action_point"
+
+
+def test_on_execute_video_end_transition_clears_execution_video_log_state(reload_module):
+    callbacks = reload_module("gradio_callbacks")
+
+    result = callbacks.on_execute_video_end_transition(
+        "uid-1",
+        {
+            "exec_btn_interactive": True,
+            "reference_action_interactive": True,
+        },
+        {
+            "preserve_terminal_log": False,
+            "terminal_log_value": None,
+            "preserve_execute_video_log": True,
+            "execute_video_log_value": "Executing Action b",
+        },
+    )
+
+    assert result[8]["value"] == callbacks.UI_TEXT["log"]["action_selection_prompt"]
+    assert result[11] == callbacks._default_post_execute_log_state()
+    assert result[12] == "action_point"
 
 
 def test_on_execute_video_end_transition_keeps_terminal_buttons_disabled(reload_module):
@@ -134,7 +157,13 @@ def test_on_execute_video_end_transition_keeps_terminal_buttons_disabled(reload_
     assert result[8]["value"] == "terminal banner"
     assert result[9]["interactive"] is False
     assert result[10]["interactive"] is True
-    assert result[11] == "action_point"
+    assert result[11] == callbacks._normalize_post_execute_log_state(
+        {
+            "preserve_terminal_log": True,
+            "terminal_log_value": "terminal banner",
+        }
+    )
+    assert result[12] == "action_point"
 
 
 def test_on_option_select_preserves_terminal_log_state(reload_module):
@@ -158,6 +187,36 @@ def test_on_option_select_preserves_terminal_log_state(reload_module):
     assert log_state == {
         "preserve_terminal_log": True,
         "terminal_log_value": "episode success banner",
+        "preserve_execute_video_log": False,
+        "execute_video_log_value": None,
+    }
+
+
+def test_on_option_select_preserves_execution_video_log_state(reload_module):
+    callbacks = reload_module("gradio_callbacks")
+
+    coords_update, img_update, log_update, suppress_flag, log_state = callbacks.on_option_select(
+        "uid-1",
+        1,
+        callbacks.UI_TEXT["coords"]["select_point"],
+        False,
+        {
+            "preserve_terminal_log": False,
+            "terminal_log_value": None,
+            "preserve_execute_video_log": True,
+            "execute_video_log_value": "Executing Action b",
+        },
+    )
+
+    assert coords_update.get("__type__") == "update"
+    assert img_update.get("__type__") == "update"
+    assert log_update["value"] == "Executing Action b"
+    assert suppress_flag is False
+    assert log_state == {
+        "preserve_terminal_log": False,
+        "terminal_log_value": None,
+        "preserve_execute_video_log": True,
+        "execute_video_log_value": "Executing Action b",
     }
 
 

gradio-web/ui_layout.py

@@ -969,6 +969,8 @@ def create_ui_blocks():
             value={
                 "preserve_terminal_log": False,
                 "terminal_log_value": None,
+                "preserve_execute_video_log": False,
+                "execute_video_log_value": None,
             }
         )
         current_task_env_state = gr.State(value=None)
@@ -1425,6 +1427,7 @@ def create_ui_blocks():
                 log_output,
                 reference_action_btn,
                 task_hint_display,
+                post_execute_log_state,
                 ui_phase_state,
             ],
             queue=False,
@@ -1451,6 +1454,7 @@ def create_ui_blocks():
                 log_output,
                 reference_action_btn,
                 task_hint_display,
+                post_execute_log_state,
                 ui_phase_state,
             ],
             queue=False,