Restructure app to match reference visualizer: sidebar, video player, synced plots

app.py

@@ -5,7 +5,11 @@ Egocentric hand tracking dataset visualizer for robot training data
 
 import gradio as gr
 import json
+import numpy as np
 from pathlib import Path
+import plotly.graph_objects as go
+import plotly.io as pio
+from typing import Dict, List
 
 # Load pipeline data
 DATA_DIR = Path(__file__).parent / "data"
@@ -43,185 +47,316 @@ for frame_data in end_effector.values():
 
 print(f"Stats: frames={total_frames}, left={left_poses}, right={right_poses}")
 
-def get_frame_info(frame_idx):
-    """Get info for a specific frame."""
-    try:
-        frame_key = str(int(frame_idx))
-        
-        # Hand detection status
-        hand_data = hands_2d.get(frame_key) or {}
-        left_detected = bool(hand_data.get('left_hand'))
-        right_detected = bool(hand_data.get('right_hand'))
+# Video path
+video_path = DATA_DIR / "video.mp4"
+
+# Prepare time-series data
+def prepare_data():
+    """Prepare time-series data for plots."""
+    times = []
+    left_pos = {'x': [], 'y': [], 'z': []}
+    left_rot = {'yaw': [], 'pitch': [], 'roll': []}
+    right_pos = {'x': [], 'y': [], 'z': []}
+    right_rot = {'yaw': [], 'pitch': [], 'roll': []}
+    
+    frame_keys = sorted([int(k) for k in end_effector.keys() if k.isdigit()])
+    
+    for frame_idx in frame_keys:
+        frame_key = str(frame_idx)
+        t = frame_idx / fps
+        times.append(t)
         
-        # End effector pose
-        ee_data = end_effector.get(frame_key) or {}
-        left_hand_data = ee_data.get('left_hand') if ee_data else None
-        right_hand_data = ee_data.get('right_hand') if ee_data else None
+        ee_data = end_effector.get(frame_key, {}) or {}
         
-        left_pose = None
-        right_pose = None
+        # Left hand
+        left_hand_data = ee_data.get('left_hand')
         if left_hand_data and isinstance(left_hand_data, dict):
-            left_pose = left_hand_data.get('pose_6dof')
-        if right_hand_data and isinstance(right_hand_data, dict):
-            right_pose = right_hand_data.get('pose_6dof')
-        
-        # Action
-        action_data = actions.get(frame_key) or {}
-        left_action = action_data.get('left_hand_action') if action_data else None
-        camera_action = action_data.get('camera_action') if action_data else None
+            pose = left_hand_data.get('pose_6dof')
+            if pose and len(pose) >= 6:
+                left_pos['x'].append(pose[0] * 100)  # m to cm
+                left_pos['y'].append(pose[1] * 100)
+                left_pos['z'].append(pose[2] * 100)
+                left_rot['roll'].append(pose[3] * 57.3)  # rad to deg
+                left_rot['pitch'].append(pose[4] * 57.3)
+                left_rot['yaw'].append(pose[5] * 57.3)
+            else:
+                for k in left_pos: left_pos[k].append(None)
+                for k in left_rot: left_rot[k].append(None)
+        else:
+            for k in left_pos: left_pos[k].append(None)
+            for k in left_rot: left_rot[k].append(None)
         
-        # Format output
-        info = f"""### Frame {int(frame_idx)} / {total_frames - 1}
-**Time:** {int(frame_idx) / fps:.2f}s
+        # Right hand
+        right_hand_data = ee_data.get('right_hand')
+        if right_hand_data and isinstance(right_hand_data, dict):
+            pose = right_hand_data.get('pose_6dof')
+            if pose and len(pose) >= 6:
+                right_pos['x'].append(pose[0] * 100)
+                right_pos['y'].append(pose[1] * 100)
+                right_pos['z'].append(pose[2] * 100)
+                right_rot['roll'].append(pose[3] * 57.3)
+                right_rot['pitch'].append(pose[4] * 57.3)
+                right_rot['yaw'].append(pose[5] * 57.3)
+            else:
+                for k in right_pos: right_pos[k].append(None)
+                for k in right_rot: right_rot[k].append(None)
+        else:
+            for k in right_pos: right_pos[k].append(None)
+            for k in right_rot: right_rot[k].append(None)
+    
+    return {
+        'times': times,
+        'left_pos': left_pos,
+        'left_rot': left_rot,
+        'right_pos': right_pos,
+        'right_rot': right_rot
+    }
 
----
+plot_data = prepare_data()
 
-#### 🖐️ Hand Detection
-- **Left Hand:** {'✅ Detected' if left_detected else '❌ Not Detected'}
-- **Right Hand:** {'✅ Detected' if right_detected else '❌ Not Detected'}
+METRIC_LABELS = {
+    "x_cm": "X (cm)",
+    "y_cm": "Y (cm)",
+    "z_cm": "Z (cm)",
+    "yaw_deg": "Yaw (°)",
+    "pitch_deg": "Pitch (°)",
+    "roll_deg": "Roll (°)",
+}
 
----
+PLOT_GRID = [
+    ["x_cm", "y_cm", "z_cm"],
+    ["yaw_deg", "pitch_deg", "roll_deg"],
+]
 
-#### 📍 End-Effector Pose (6DoF)
-"""
-        
-        if left_pose and len(left_pose) >= 6:
-            info += f"""
-**Left Hand:**
-- Position: X={left_pose[0]*100:.1f}cm, Y={left_pose[1]*100:.1f}cm, Z={left_pose[2]*100:.1f}cm
-- Rotation: Roll={left_pose[3]*57.3:.1f}°, Pitch={left_pose[4]*57.3:.1f}°, Yaw={left_pose[5]*57.3:.1f}°
-"""
-        else:
-            info += "\n**Left Hand:** No pose available\n"
-        
-        if right_pose and len(right_pose) >= 6:
-            info += f"""
-**Right Hand:**
-- Position: X={right_pose[0]*100:.1f}cm, Y={right_pose[1]*100:.1f}cm, Z={right_pose[2]*100:.1f}cm
-- Rotation: Roll={right_pose[3]*57.3:.1f}°, Pitch={right_pose[4]*57.3:.1f}°, Yaw={right_pose[5]*57.3:.1f}°
+PLOT_ORDER = [metric for row in PLOT_GRID for metric in row]
+
+CUSTOM_CSS = """
+:root, .gradio-container, body {
+    background-color: #050a18 !important;
+    color: #f8fafc !important;
+    font-family: 'Inter', 'Segoe UI', system-ui, sans-serif;
+}
+.side-panel {
+    background: #0f172a;
+    padding: 20px;
+    border-radius: 18px;
+    border: 1px solid #1f2b47;
+    min-height: 100%;
+}
+.stats-card ul {
+    list-style: none;
+    padding: 0;
+    margin: 0;
+    font-size: 0.92rem;
+}
+.stats-card li {
+    margin-bottom: 10px;
+    color: #e2e8f0;
+}
+.stats-card span {
+    display: inline-block;
+    margin-right: 6px;
+    color: #7dd3fc;
+}
+.main-panel {
+    padding-top: 8px;
+}
+.instruction-card {
+    background: #0f172a;
+    padding: 18px 20px;
+    border-radius: 18px;
+    border: 1px solid #1f2b47;
+}
+.instruction-label {
+    font-size: 0.75rem;
+    letter-spacing: 0.12em;
+    text-transform: uppercase;
+    color: #94a3b8;
+    margin-bottom: 10px;
+}
+.instruction-text {
+    font-size: 1.1rem;
+    line-height: 1.5;
+}
+.video-card {
+    background: #0f172a;
+    border: 1px solid #1f2b47;
+    border-radius: 18px;
+    padding: 18px 20px;
+    margin-top: 18px;
+}
+.video-title {
+    font-size: 0.78rem;
+    text-transform: uppercase;
+    letter-spacing: 0.18em;
+    color: #94a3b8;
+    margin-bottom: 8px;
+}
+.video-panel video {
+    border-radius: 12px;
+    border: 1px solid #1f2b47;
+    background: #030712;
+}
+.download-button button {
+    border-radius: 999px;
+    border: 1px solid #334155;
+    background: #1e293b;
+    color: #f8fafc;
+    font-size: 0.85rem;
+    padding: 8px 24px;
+}
+.download-button button:hover {
+    border-color: #67e8f9;
+    color: #67e8f9;
+}
+.plots-wrap {
+    margin-top: 18px;
+}
+.plots-wrap .gr-row {
+    gap: 16px;
+}
+.plot-html {
+    background: #111a2c;
+    border-radius: 12px;
+    padding: 10px;
+    border: 1px solid #1f2b47;
+    min-height: 320px;
+}
+.plot-html iframe {
+    width: 100%;
+    height: 300px;
+    border: none;
+}
 """
-        
-        info += "\n---\n\n#### 🎯 Actions (Delta per frame)\n"
-        
-        if left_action and len(left_action) >= 3:
-            mag = (left_action[0]**2 + left_action[1]**2 + left_action[2]**2)**0.5 * 100
-            info += f"**Left Hand Movement:** {mag:.2f} cm\n"
-        
-        if camera_action and len(camera_action) >= 3:
-            cam_mag = (camera_action[0]**2 + camera_action[1]**2 + camera_action[2]**2)**0.5 * 100
-            info += f"**Camera Movement:** {cam_mag:.2f} cm\n"
-        
-        return info
-    except Exception as e:
-        return f"Error: {str(e)}"
-
-def get_frame_image(frame_idx):
-    """Get RGB frame image path."""
-    # Round to nearest 10 (we only have every 10th frame)
-    idx = int(frame_idx)
-    idx = (idx // 10) * 10
-    frame_path = DATA_DIR / "frames" / f"{idx}.jpg"
-    if frame_path.exists():
-        return str(frame_path)
-    # Try exact frame
-    frame_path = DATA_DIR / "frames" / f"{int(frame_idx)}.jpg"
-    if frame_path.exists():
-        return str(frame_path)
-    return None
-
-def update_display(frame_idx):
-    """Update frame display."""
-    img = get_frame_image(frame_idx)
-    info = get_frame_info(frame_idx)
-    return img, info
-
-# Build Gradio Interface
-with gr.Blocks(title="DI Human Demo Visualizer") as demo:
-    
-    # Header
-    gr.Markdown("""
-    # 🤖 Dynamic Intelligence - Human Demo Visualizer
-    
-    **Egocentric hand tracking dataset for humanoid robot training**
-    
-    Pipeline: iPhone LiDAR → MediaPipe → 6DoF End-Effector → Robot Training Data
-    """)
-    
-    # Stats row
-    gr.Markdown(f"""
-    | Stat | Value |
-    |------|-------|
-    | Total Frames | {total_frames} |
-    | Hand Detection | {hand_detection_rate:.1f}% |
-    | Left Hand Poses | {left_poses} |
-    | Right Hand Poses | {right_poses} |
-    | FPS | {fps} |
-    """)
+
+def build_plot_fig(metric: str, hand: str = "left") -> go.Figure:
+    """Build Plotly figure for a metric."""
+    times = plot_data['times']
     
-    gr.Markdown("---")
+    if hand == "left":
+        if "cm" in metric:
+            data = plot_data['left_pos'][metric.replace('_cm', '')]
+        else:
+            data = plot_data['left_rot'][metric.replace('_deg', '')]
+        name = "Left Hand"
+    else:
+        if "cm" in metric:
+            data = plot_data['right_pos'][metric.replace('_cm', '')]
+        else:
+            data = plot_data['right_rot'][metric.replace('_deg', '')]
+        name = "Right Hand"
     
-    # Main content
-    with gr.Row():
-        with gr.Column(scale=2):
-            gr.Markdown("### 📹 RGB Frame")
-            frame_image = gr.Image(label="Frame", height=400)
-            frame_slider = gr.Slider(
-                minimum=0,
-                maximum=max(1, total_frames - 1),
-                step=10,
-                value=0,
-                label="Frame (every 10th frame available)"
+    fig = go.Figure()
+    fig.add_trace(
+        go.Scatter(
+            x=times,
+            y=data,
+            mode="lines",
+            name=name,
+            line=dict(color="#67e8f9", width=2)
+        )
+    )
+    fig.update_layout(
+        margin=dict(l=20, r=20, t=30, b=20),
+        height=250,
+        template="plotly_dark",
+        legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1),
+        xaxis_title="Time (s)",
+        yaxis_title=METRIC_LABELS[metric],
+    )
+    fig.update_xaxes(showgrid=True, gridwidth=0.5, gridcolor="rgba(255,255,255,0.1)")
+    fig.update_yaxes(showgrid=True, gridwidth=0.5, gridcolor="rgba(255,255,255,0.1)")
+    return fig
+
+def build_plot_html(metric: str, hand: str = "left") -> str:
+    """Build Plotly HTML for a metric."""
+    fig = build_plot_fig(metric, hand)
+    return pio.to_html(fig, include_plotlyjs="cdn", full_html=False)
+
+# Build interface
+stats_html = f"""
+<div class="stats-card">
+  <ul>
+    <li><span>Number of samples/frames:</span> {total_frames:,}</li>
+    <li><span>Hand detection rate:</span> {hand_detection_rate:.1f}%</li>
+    <li><span>Left hand poses:</span> {left_poses}</li>
+    <li><span>Right hand poses:</span> {right_poses}</li>
+    <li><span>Frames per second:</span> {fps:.1f}</li>
+  </ul>
+</div>
+"""
+
+instruction_text = "LiDAR-based egocentric hand tracking for robot training data"
+
+theme = gr.themes.Soft(
+    primary_hue="cyan", secondary_hue="blue", neutral_hue="slate"
+).set(
+    body_background_fill="#0c1424",
+    body_text_color="#f8fafc",
+    block_background_fill="#111a2c",
+    block_title_text_color="#f8fafc",
+    input_background_fill="#151f33",
+    border_color_primary="#1f2b47",
+    shadow_drop="none",
+)
+
+def format_instruction_html(text: str) -> str:
+    import html
+    safe_text = html.escape(text)
+    return (
+        '<div class="instruction-card">'
+        '<p class="instruction-label">Language Instruction</p>'
+        f'<p class="instruction-text">{safe_text}</p>'
+        "</div>"
+    )
+
+with gr.Blocks(theme=theme, css=CUSTOM_CSS) as demo:
+    gr.Markdown("# 🤖 Dynamic Intelligence - Human Demo Visualizer")
+    gr.Markdown(
+        "Egocentric hand tracking dataset for humanoid robot training. "
+        "Pipeline: iPhone LiDAR → MediaPipe → 6DoF End-Effector → Robot Training Data"
+    )
+
+    with gr.Row(equal_height=True):
+        with gr.Column(scale=1, min_width=260, elem_classes=["side-panel"]):
+            gr.HTML(stats_html)
+        with gr.Column(scale=2, min_width=640, elem_classes=["main-panel"]):
+            instruction_box = gr.HTML(
+                format_instruction_html(instruction_text),
+                label="Language Instruction",
             )
-        
-        with gr.Column(scale=1):
-            frame_info = gr.Markdown("Select a frame to see details")
-    
-    gr.Markdown("---")
-    
-    # Plots
-    gr.Markdown("### 📊 Validation Plots")
-    
-    with gr.Row():
-        camera_plot = DATA_DIR / "plots" / "camera_trajectory.png"
-        left_plot = DATA_DIR / "plots" / "left_hand_trajectory.png"
-        if camera_plot.exists():
-            gr.Image(value=str(camera_plot), label="Camera Trajectory")
-        if left_plot.exists():
-            gr.Image(value=str(left_plot), label="Left Hand Trajectory")
-    
-    with gr.Row():
-        pose_plot = DATA_DIR / "plots" / "hand_pose_vs_time.png"
-        action_plot = DATA_DIR / "plots" / "actions_histogram.png"
-        if pose_plot.exists():
-            gr.Image(value=str(pose_plot), label="Hand Pose vs Time")
-        if action_plot.exists():
-            gr.Image(value=str(action_plot), label="Actions Distribution")
-    
-    # Physics validation
-    gr.Markdown("""
-    ---
-    
-    ### ✅ Physics Validation Results
-    
-    | Check | Status | Details |
-    |-------|--------|---------|
-    | Camera Trajectory | ✅ PASS | Smooth movement, ~40cm total range |
-    | Hand Depth Range | ✅ PASS | 15-60cm from camera (realistic) |
-    | Action Magnitudes | ✅ PASS | Median 0.34cm/frame (no tracking errors) |
-    | 6DoF Rotations | ✅ PASS | Natural hand movement patterns |
-    
-    ---
-    
-    **Organization:** [Dynamic Intelligence](https://huggingface.co/DynamicIntelligence)
-    """)
+            with gr.Column(elem_classes=["video-card"]):
+                gr.HTML('<div class="video-title">RGB Video</div>')
+                video = gr.Video(
+                    height=360,
+                    value=str(video_path) if video_path.exists() else None,
+                    elem_classes=["video-panel"],
+                    show_label=False,
+                    show_download_button=False,
+                )
+                download_button = gr.DownloadButton(
+                    label="Download",
+                    value=str(video_path) if video_path.exists() else None,
+                    elem_classes=["download-button"],
+                )
+
+    gr.Markdown("### Left Hand Trajectories", elem_classes=["plots-title"])
+    plot_outputs_left = []
+    with gr.Column(elem_classes=["plots-wrap"]):
+        for row in PLOT_GRID:
+            with gr.Row():
+                for metric in row:
+                    plot = gr.HTML(value=build_plot_html(metric, "left"), elem_classes=["plot-html"])
+                    plot_outputs_left.append(plot)
     
-    # Event handler
-    frame_slider.change(
-        fn=update_display,
-        inputs=[frame_slider],
-        outputs=[frame_image, frame_info]
-    )
+    gr.Markdown("### Right Hand Trajectories", elem_classes=["plots-title"])
+    plot_outputs_right = []
+    with gr.Column(elem_classes=["plots-wrap"]):
+        for row in PLOT_GRID:
+            with gr.Row():
+                for metric in row:
+                    plot = gr.HTML(value=build_plot_html(metric, "right"), elem_classes=["plot-html"])
+                    plot_outputs_right.append(plot)
 
-# Launch
 if __name__ == "__main__":
-    demo.launch()
+    demo.queue().launch(show_api=False)