fix video display v1

gradio-web/config.py

@@ -40,7 +40,7 @@ DEMO_VIDEO_ENV_IDS = [
 
 UI_TEXT = {
     "log": {
-        "action_selection_prompt": "please select the action in the right 👈,\nsome actions also need to select keypoint",
+        "action_selection_prompt": "please select the action in the left 👈,\nsome actions also need to select keypoint",
         "demo_video_prompt": 'press "Watch Video Input🎬" to watch a video\nNote: you can only watch the video once',
         "session_error": "Session Error",
         "reference_action_error": "Ground Truth Action Error: {error}",

gradio-web/test/test_ui_phase_machine_runtime_e2e.py

@@ -100,6 +100,29 @@ def _read_coords_box_value(page) -> str | None:
     )
 
 
+def _read_phase_visibility(page) -> dict[str, bool | str | None]:
+    return page.evaluate(
+        """() => {
+            const visible = (id) => {
+                const el = document.getElementById(id);
+                if (!el) return false;
+                const st = getComputedStyle(el);
+                return st.display !== 'none' && st.visibility !== 'hidden' && el.getClientRects().length > 0;
+            };
+            const videoEl = document.querySelector('#demo_video video');
+            return {
+                videoPhase: visible('video_phase_group'),
+                video: visible('demo_video'),
+                actionPhase: visible('action_phase_group'),
+                action: visible('live_obs'),
+                controlPhase: visible('control_panel_group'),
+                control: visible('action_radio'),
+                currentSrc: videoEl ? videoEl.currentSrc : null,
+            };
+        }"""
+    )
+
+
 def _read_live_obs_geometry(page) -> dict[str, dict[str, float] | None]:
     return page.evaluate(
         """() => {
@@ -1051,6 +1074,172 @@ def test_header_task_env_normalization_and_fallback(monkeypatch, task_info_text,
         demo.close()
 
 
+def test_header_task_switch_to_video_task_shows_demo_phase(monkeypatch):
+    ui_layout = importlib.reload(importlib.import_module("ui_layout"))
+
+    fake_obs = np.zeros((24, 24, 3), dtype=np.uint8)
+    fake_obs_img = Image.fromarray(fake_obs)
+    demo_video_path = gr.get_video("world.mp4")
+    switch_calls = []
+
+    def fake_init_app(request=None):
+        _ = request
+        return (
+            "uid-header-video",
+            gr.update(visible=True),  # main_interface
+            gr.update(value=fake_obs_img, interactive=False),  # img_display
+            "ready",  # log_output
+            gr.update(choices=[("pick", 0)], value=None),  # options_radio
+            "goal",  # goal_box
+            "No need for coordinates",  # coords_box
+            gr.update(value=None, visible=False),  # video_display
+            "PickXtimes (Episode 1)",  # task_info_box
+            "Completed: 0",  # progress_info_box
+            gr.update(interactive=True),  # restart_episode_btn
+            gr.update(interactive=True),  # next_task_btn
+            gr.update(interactive=True),  # exec_btn
+            gr.update(visible=False),  # video_phase_group
+            gr.update(visible=True),  # action_phase_group
+            gr.update(visible=True),  # control_panel_group
+            gr.update(value="hint"),  # task_hint_display
+            gr.update(visible=False),  # loading_overlay
+            gr.update(interactive=True),  # reference_action_btn
+        )
+
+    def fake_switch_env_wrapper(uid, selected_env):
+        switch_calls.append((uid, selected_env))
+        return (
+            uid,
+            gr.update(visible=True),  # main_interface
+            gr.update(value=fake_obs_img, interactive=False),  # img_display
+            "demo prompt",  # log_output
+            gr.update(choices=[("pick", 0)], value=None),  # options_radio
+            "video goal",  # goal_box
+            "No need for coordinates",  # coords_box
+            gr.update(value=demo_video_path, visible=True),  # video_display
+            "VideoPlaceButton (Episode 1)",  # task_info_box
+            "Completed: 0",  # progress_info_box
+            gr.update(interactive=True),  # restart_episode_btn
+            gr.update(interactive=True),  # next_task_btn
+            gr.update(interactive=True),  # exec_btn
+            gr.update(visible=True),  # video_phase_group
+            gr.update(visible=False),  # action_phase_group
+            gr.update(visible=False),  # control_panel_group
+            gr.update(value="video hint"),  # task_hint_display
+            gr.update(visible=False),  # loading_overlay
+            gr.update(interactive=True),  # reference_action_btn
+        )
+
+    monkeypatch.setattr(ui_layout, "init_app", fake_init_app)
+    monkeypatch.setattr(ui_layout, "switch_env_wrapper", fake_switch_env_wrapper)
+    monkeypatch.setattr(ui_layout.user_manager, "env_choices", ["PickXtimes", "VideoPlaceButton"])
+
+    demo = ui_layout.create_ui_blocks()
+
+    port = _free_port()
+    host = "127.0.0.1"
+    root_url = f"http://{host}:{port}/"
+
+    app = FastAPI(title="header-task-switch-video-phase-test")
+    app = gr.mount_gradio_app(app, demo, path="/")
+
+    config = uvicorn.Config(app, host=host, port=port, log_level="error")
+    server = uvicorn.Server(config)
+    thread = threading.Thread(target=server.run, daemon=True)
+    thread.start()
+    _wait_http_ready(root_url)
+
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page(viewport={"width": 1280, "height": 900})
+            page.goto(root_url, wait_until="domcontentloaded")
+            page.wait_for_selector("#main_interface_root", state="visible", timeout=15000)
+            page.wait_for_function(
+                """() => {
+                    const root = document.getElementById('header_task');
+                    const input = root ? root.querySelector('input') : null;
+                    return !!input && input.value.trim() === 'PickXtimes';
+                }""",
+                timeout=5000,
+            )
+
+            page.click("#header_task input")
+            page.get_by_role("option", name="VideoPlaceButton").click()
+
+            page.wait_for_function(
+                """() => {
+                    const visible = (id) => {
+                        const el = document.getElementById(id);
+                        if (!el) return false;
+                        const st = getComputedStyle(el);
+                        return st.display !== 'none' && st.visibility !== 'hidden' && el.getClientRects().length > 0;
+                    };
+                    const videoEl = document.querySelector('#demo_video video');
+                    return (
+                        visible('video_phase_group') &&
+                        visible('demo_video') &&
+                        !visible('action_phase_group') &&
+                        !visible('control_panel_group') &&
+                        !!(videoEl && videoEl.currentSrc)
+                    );
+                }""",
+                timeout=10000,
+            )
+
+            phase_after_switch = _read_phase_visibility(page)
+            assert phase_after_switch["videoPhase"] is True
+            assert phase_after_switch["video"] is True
+            assert phase_after_switch["actionPhase"] is False
+            assert phase_after_switch["controlPhase"] is False
+            assert phase_after_switch["currentSrc"]
+            assert switch_calls == [("uid-header-video", "VideoPlaceButton")]
+
+            did_dispatch_end = page.evaluate(
+                """() => {
+                    const videoEl = document.querySelector('#demo_video video');
+                    if (!videoEl) return false;
+                    videoEl.dispatchEvent(new Event('ended', { bubbles: true }));
+                    return true;
+                }"""
+            )
+            assert did_dispatch_end
+
+            page.wait_for_function(
+                """() => {
+                    const visible = (id) => {
+                        const el = document.getElementById(id);
+                        if (!el) return false;
+                        const st = getComputedStyle(el);
+                        return st.display !== 'none' && st.visibility !== 'hidden' && el.getClientRects().length > 0;
+                    };
+                    return (
+                        !visible('video_phase_group') &&
+                        !visible('demo_video') &&
+                        visible('action_phase_group') &&
+                        visible('control_panel_group') &&
+                        visible('live_obs') &&
+                        visible('action_radio')
+                    );
+                }""",
+                timeout=5000,
+            )
+
+            phase_after_end = _read_phase_visibility(page)
+            assert phase_after_end["videoPhase"] is False
+            assert phase_after_end["video"] is False
+            assert phase_after_end["actionPhase"] is True
+            assert phase_after_end["action"] is True
+            assert phase_after_end["controlPhase"] is True
+            assert phase_after_end["control"] is True
+
+            browser.close()
+    finally:
+        server.should_exit = True
+        thread.join(timeout=10)
+        demo.close()
+
+
 def test_phase_machine_runtime_local_video_path_end_transition():
     import gradio_callbacks as cb
 

gradio-web/ui_layout.py

@@ -290,6 +290,30 @@ def _with_phase_from_load(load_result):
     return (*load_result, phase)
 
 
+def _skip_load_flow():
+    return tuple(gr.skip() for _ in range(20))
+
+
+def _phase_visibility_updates(phase):
+    if phase == PHASE_DEMO_VIDEO:
+        return (
+            gr.update(visible=True),
+            gr.update(visible=False),
+            gr.update(visible=False),
+        )
+    if phase in {PHASE_ACTION_KEYPOINT, PHASE_EXECUTION_PLAYBACK}:
+        return (
+            gr.update(visible=False),
+            gr.update(visible=True),
+            gr.update(visible=True),
+        )
+    return (
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+    )
+
+
 def create_ui_blocks():
     """构建 Gradio Blocks，并完成页面阶段状态（phase）的联动绑定。"""
 
@@ -337,6 +361,8 @@ def create_ui_blocks():
 
         uid_state = gr.State(value=None)
         ui_phase_state = gr.State(value=PHASE_INIT)
+        pending_header_task_state = gr.State(value=None)
+        programmatic_header_task_state = gr.State(value=None)
         live_obs_timer = gr.Timer(value=1.0 / LIVE_OBS_REFRESH_HZ, active=True)
 
         task_info_box = gr.Textbox(visible=False, elem_id="task_info_box")
@@ -450,6 +476,34 @@ def create_ui_blocks():
                                 elem_id="task_hint_display",
                             )
 
+        load_flow_outputs = [
+            uid_state,
+            main_interface,
+            img_display,
+            log_output,
+            options_radio,
+            goal_box,
+            coords_box,
+            video_display,
+            task_info_box,
+            progress_info_box,
+            restart_episode_btn,
+            next_task_btn,
+            exec_btn,
+            video_phase_group,
+            action_phase_group,
+            control_panel_group,
+            task_hint_display,
+            loading_overlay,
+            reference_action_btn,
+            ui_phase_state,
+        ]
+        phase_visibility_outputs = [
+            video_phase_group,
+            action_phase_group,
+            control_panel_group,
+        ]
+
         def _normalize_env_choice(env_value, choices):
             if env_value is None:
                 return None
@@ -463,7 +517,7 @@ def create_ui_blocks():
                     lower_map.setdefault(choice_text.lower(), choice_text)
             return lower_map.get(env_text.lower(), env_text)
 
-        def _build_header_task_update(task_text, fallback_env=None):
+        def _resolve_header_task_state(task_text, fallback_env=None):
             base_choices = list(user_manager.env_choices)
             parsed_env = render_header_task(task_text)
             selected_env = _normalize_env_choice(parsed_env, base_choices)
@@ -473,13 +527,23 @@ def create_ui_blocks():
             choices = list(base_choices)
             if selected_env and selected_env not in choices:
                 choices.append(selected_env)
+            return choices, selected_env
+
+        def _build_header_task_update(task_text, fallback_env=None):
+            choices, selected_env = _resolve_header_task_state(task_text, fallback_env=fallback_env)
             return gr.update(choices=choices, value=selected_env)
 
         def sync_header_from_task(task_text, goal_text):
-            return _build_header_task_update(task_text), render_header_goal(goal_text)
+            _, selected_env = _resolve_header_task_state(task_text)
+            return _build_header_task_update(task_text), render_header_goal(goal_text), selected_env
 
         def sync_header_from_goal(goal_text, task_text, current_header_task):
-            return _build_header_task_update(task_text, fallback_env=current_header_task), render_header_goal(goal_text)
+            _, selected_env = _resolve_header_task_state(task_text, fallback_env=current_header_task)
+            return (
+                _build_header_task_update(task_text, fallback_env=current_header_task),
+                render_header_goal(goal_text),
+                selected_env,
+            )
 
         def init_app_with_phase(request: gr.Request):
             return _with_phase_from_load(init_app(request))
@@ -493,108 +557,87 @@ def create_ui_blocks():
         def switch_env_with_phase(uid, selected_env):
             return _with_phase_from_load(switch_env_wrapper(uid, selected_env))
 
+        def maybe_switch_env_with_phase(uid, selected_env):
+            if not selected_env:
+                return _skip_load_flow()
+            return switch_env_with_phase(uid, selected_env)
+
+        def prepare_header_task_switch(selected_env, programmatic_selected_env):
+            base_choices = list(user_manager.env_choices)
+            normalized_selected_env = _normalize_env_choice(selected_env, base_choices)
+            normalized_programmatic_env = _normalize_env_choice(programmatic_selected_env, base_choices)
+
+            if not normalized_selected_env:
+                return None, None, gr.update(visible=False)
+            if normalized_selected_env == normalized_programmatic_env:
+                return None, None, gr.update(visible=False)
+            return normalized_selected_env, None, show_loading_info()
+
         task_info_box.change(
             fn=sync_header_from_task,
             inputs=[task_info_box, goal_box],
-            outputs=[header_task_box, header_goal_box],
+            outputs=[header_task_box, header_goal_box, programmatic_header_task_state],
         )
         goal_box.change(
             fn=sync_header_from_goal,
             inputs=[goal_box, task_info_box, header_task_box],
-            outputs=[header_task_box, header_goal_box],
+            outputs=[header_task_box, header_goal_box, programmatic_header_task_state],
         )
 
-        header_task_box.input(fn=show_loading_info, outputs=[loading_overlay]).then(
-            fn=switch_env_with_phase,
-            inputs=[uid_state, header_task_box],
-            outputs=[
-                uid_state,
-                main_interface,
-                img_display,
-                log_output,
-                options_radio,
-                goal_box,
-                coords_box,
-                video_display,
-                task_info_box,
-                progress_info_box,
-                restart_episode_btn,
-                next_task_btn,
-                exec_btn,
-                video_phase_group,
-                action_phase_group,
-                control_panel_group,
-                task_hint_display,
-                loading_overlay,
-                reference_action_btn,
-                ui_phase_state,
-            ],
+        header_task_box.change(
+            fn=prepare_header_task_switch,
+            inputs=[header_task_box, programmatic_header_task_state],
+            outputs=[pending_header_task_state, programmatic_header_task_state, loading_overlay],
+            queue=False,
+            show_progress="hidden",
         ).then(
-            fn=sync_header_from_task,
-            inputs=[task_info_box, goal_box],
-            outputs=[header_task_box, header_goal_box],
+            fn=maybe_switch_env_with_phase,
+            inputs=[uid_state, pending_header_task_state],
+            outputs=load_flow_outputs,
+        ).then(
+            fn=_phase_visibility_updates,
+            inputs=[ui_phase_state],
+            outputs=phase_visibility_outputs,
+            queue=False,
+            show_progress="hidden",
+        ).then(
+            fn=lambda _selected_env: None,
+            inputs=[pending_header_task_state],
+            outputs=[pending_header_task_state],
+            queue=False,
+            show_progress="hidden",
         )
 
         next_task_btn.click(fn=show_loading_info, outputs=[loading_overlay]).then(
             fn=load_next_task_with_phase,
             inputs=[uid_state],
-            outputs=[
-                uid_state,
-                main_interface,
-                img_display,
-                log_output,
-                options_radio,
-                goal_box,
-                coords_box,
-                video_display,
-                task_info_box,
-                progress_info_box,
-                restart_episode_btn,
-                next_task_btn,
-                exec_btn,
-                video_phase_group,
-                action_phase_group,
-                control_panel_group,
-                task_hint_display,
-                loading_overlay,
-                reference_action_btn,
-                ui_phase_state,
-            ],
+            outputs=load_flow_outputs,
+        ).then(
+            fn=_phase_visibility_updates,
+            inputs=[ui_phase_state],
+            outputs=phase_visibility_outputs,
+            queue=False,
+            show_progress="hidden",
         ).then(
             fn=sync_header_from_task,
             inputs=[task_info_box, goal_box],
-            outputs=[header_task_box, header_goal_box],
+            outputs=[header_task_box, header_goal_box, programmatic_header_task_state],
         )
 
         restart_episode_btn.click(fn=show_loading_info, outputs=[loading_overlay]).then(
             fn=restart_episode_with_phase,
             inputs=[uid_state],
-            outputs=[
-                uid_state,
-                main_interface,
-                img_display,
-                log_output,
-                options_radio,
-                goal_box,
-                coords_box,
-                video_display,
-                task_info_box,
-                progress_info_box,
-                restart_episode_btn,
-                next_task_btn,
-                exec_btn,
-                video_phase_group,
-                action_phase_group,
-                control_panel_group,
-                task_hint_display,
-                loading_overlay,
-                reference_action_btn,
-                ui_phase_state,
-            ],
+            outputs=load_flow_outputs,
+        ).then(
+            fn=_phase_visibility_updates,
+            inputs=[ui_phase_state],
+            outputs=phase_visibility_outputs,
+            queue=False,
+            show_progress="hidden",
         ).then(
             fn=sync_header_from_task,
             inputs=[task_info_box, goal_box],
-            outputs=[header_task_box, header_goal_box],
+            outputs=[header_task_box, header_goal_box, programmatic_header_task_state],
         )
 
         video_display.end(
@@ -701,32 +744,17 @@ def create_ui_blocks():
         demo.load(
             fn=init_app_with_phase,
             inputs=[],
-            outputs=[
-                uid_state,
-                main_interface,
-                img_display,
-                log_output,
-                options_radio,
-                goal_box,
-                coords_box,
-                video_display,
-                task_info_box,
-                progress_info_box,
-                restart_episode_btn,
-                next_task_btn,
-                exec_btn,
-                video_phase_group,
-                action_phase_group,
-                control_panel_group,
-                task_hint_display,
-                loading_overlay,
-                reference_action_btn,
-                ui_phase_state,
-            ],
+            outputs=load_flow_outputs,
+        ).then(
+            fn=_phase_visibility_updates,
+            inputs=[ui_phase_state],
+            outputs=phase_visibility_outputs,
+            queue=False,
+            show_progress="hidden",
         ).then(
             fn=sync_header_from_task,
             inputs=[task_info_box, goal_box],
-            outputs=[header_task_box, header_goal_box],
+            outputs=[header_task_box, header_goal_box, programmatic_header_task_state],
         )
 
     return demo