Raffael-Kultyshev commited on
Commit
0d8a870
·
1 Parent(s): 6b2fbf0

Fix Python 3.13 compatibility - use simpler gradio version

Browse files
Files changed (3) hide show
  1. README.md +12 -30
  2. app.py +223 -304
  3. requirements.txt +1 -3
README.md CHANGED
@@ -4,51 +4,33 @@ emoji: 🤖
4
  colorFrom: blue
5
  colorTo: green
6
  sdk: gradio
7
- sdk_version: 4.44.0
8
  app_file: app.py
9
  pinned: false
10
  license: mit
 
11
  ---
12
 
13
  # Dynamic Intelligence - Trajectory Visualizer
14
 
15
- Visualize humanoid robot training data with **57 data streams** from egocentric human demonstrations.
16
 
17
- ## Data Streams (57 total)
18
 
19
  ### Visualized (15 streams)
20
- | Stream | Description | Unit |
21
- |--------|-------------|------|
22
- | Camera X, Y, Z | Camera position in world frame | meters |
23
- | Left Hand X, Y, Z | Left hand position in world frame | meters |
24
- | Right Hand X, Y, Z | Right hand position in world frame | meters |
25
- | Left Hand Roll, Pitch, Yaw | Left hand orientation | degrees |
26
- | Right Hand Roll, Pitch, Yaw | Right hand orientation | degrees |
27
 
28
  ### Stored (42 streams)
29
- - **Left hand joints:** 21 keypoints × XYZ positions
30
- - **Right hand joints:** 21 keypoints × XYZ positions
31
-
32
- ## Data Pipeline
33
-
34
- The data comes from the DI pipeline:
35
- 1. **metadata.json** → Camera poses from ARKit (world frame)
36
- 2. **hands_3d.json** → 3D hand positions and 21 joint landmarks
37
- 3. **end_effector.json** → Hand roll/pitch/yaw orientations
38
 
39
  ## Usage
40
 
41
- 1. Select an episode from the dropdown
42
- 2. Click "Load & Visualize"
43
- 3. View time series plots (15 subplots) or 3D trajectory
44
 
45
  ## Data Source
46
 
47
- Data is loaded from: [`DynamicIntelligence/humanoid-robots-training-dataset`](https://huggingface.co/datasets/DynamicIntelligence/humanoid-robots-training-dataset)
48
-
49
- ## Technical Details
50
-
51
- - Built with Gradio + Plotly
52
- - Real-time data loading from HuggingFace Hub
53
- - Interactive 3D visualization
54
- - Frame-level temporal analysis
 
4
  colorFrom: blue
5
  colorTo: green
6
  sdk: gradio
7
+ sdk_version: 4.31.0
8
  app_file: app.py
9
  pinned: false
10
  license: mit
11
+ python_version: 3.10
12
  ---
13
 
14
  # Dynamic Intelligence - Trajectory Visualizer
15
 
16
+ Visualize **57 data streams** from humanoid robot training data.
17
 
18
+ ## Data Streams
19
 
20
  ### Visualized (15 streams)
21
+ - **Camera**: X, Y, Z in world frame (meters)
22
+ - **Left Hand**: X, Y, Z position + Roll, Pitch, Yaw
23
+ - **Right Hand**: X, Y, Z position + Roll, Pitch, Yaw
 
 
 
 
24
 
25
  ### Stored (42 streams)
26
+ - **Joint positions**: 21 keypoints × 2 hands × XYZ
 
 
 
 
 
 
 
 
27
 
28
  ## Usage
29
 
30
+ 1. Click "Visualize" to load sample data
31
+ 2. View 15 time series plots showing all motion streams
32
+ 3. Explore 3D trajectory visualization
33
 
34
  ## Data Source
35
 
36
+ Data from: [DynamicIntelligence/humanoid-robots-training-dataset](https://huggingface.co/datasets/DynamicIntelligence/humanoid-robots-training-dataset)
 
 
 
 
 
 
 
app.py CHANGED
@@ -1,397 +1,316 @@
1
  #!/usr/bin/env python3
2
  """
3
- DI Trajectory Visualizer - HuggingFace Space
4
- Visualize 57 data streams from humanoid robot training data.
5
  """
6
 
7
- import gradio as gr
8
- from pathlib import Path
9
- from huggingface_hub import hf_hub_download, list_repo_files
10
- import plotly.graph_objects as go
11
- from plotly.subplots import make_subplots
12
  import json
13
  import numpy as np
 
14
  from dataclasses import dataclass
15
- from typing import Optional, List, Dict
16
 
17
- # HuggingFace dataset repo
18
- DATASET_REPO = "DynamicIntelligence/humanoid-robots-training-dataset"
 
 
 
 
19
 
 
 
20
 
21
  @dataclass
22
  class TrajectoryData:
23
- """Container for all 57 data streams."""
24
  timestamps: np.ndarray
25
- # Camera world frame (3 streams)
26
  camera_x: np.ndarray
27
  camera_y: np.ndarray
28
  camera_z: np.ndarray
29
- # Left hand position in world frame (3 streams)
30
  left_hand_x: np.ndarray
31
  left_hand_y: np.ndarray
32
  left_hand_z: np.ndarray
33
- # Right hand position in world frame (3 streams)
34
  right_hand_x: np.ndarray
35
  right_hand_y: np.ndarray
36
  right_hand_z: np.ndarray
37
- # Left hand orientation (3 streams)
38
  left_hand_roll: np.ndarray
39
  left_hand_pitch: np.ndarray
40
  left_hand_yaw: np.ndarray
41
- # Right hand orientation (3 streams)
42
  right_hand_roll: np.ndarray
43
  right_hand_pitch: np.ndarray
44
  right_hand_yaw: np.ndarray
45
- # Joint positions (42 streams - stored but not visualized)
46
- left_hand_joints: np.ndarray
47
- right_hand_joints: np.ndarray
48
 
49
 
50
- def load_trajectory_data(episode_path: Path) -> TrajectoryData:
51
- """Load all pipeline outputs for one episode."""
52
-
53
- # Load metadata.json for camera poses
54
- metadata_path = episode_path / "extracted" / "metadata.json"
55
- if not metadata_path.exists():
56
- metadata_path = episode_path / "metadata.json"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
 
58
- with open(metadata_path, 'r') as f:
59
- metadata = json.load(f)
60
-
61
- # Load hands_3d.json for hand positions and joints
62
- hands_3d_path = episode_path / "hands_3d.json"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  hands_3d = {"frames": []}
64
- if hands_3d_path.exists():
65
- with open(hands_3d_path, 'r') as f:
 
66
  hands_3d = json.load(f)
67
 
68
- # Load end_effector.json for hand orientations
69
- end_effector_path = episode_path / "end_effector.json"
70
  end_effector = {"frames": []}
71
- if end_effector_path.exists():
72
- with open(end_effector_path, 'r') as f:
 
73
  end_effector = json.load(f)
74
 
75
- # Parse timestamps
76
  frames = metadata.get('frames', metadata.get('poses', []))
77
- num_frames = len(frames)
78
  fps = metadata.get('fps', 30)
79
- timestamps = np.arange(num_frames) / fps
80
 
81
- # Parse camera world frame positions
 
 
82
  camera_x, camera_y, camera_z = [], [], []
83
  for f in frames:
84
- if 'camera_pose' in f:
85
- pos = f['camera_pose'].get('position', [0, 0, 0])
86
- elif 'position' in f:
87
- pos = f['position']
88
- else:
89
- pos = [0, 0, 0]
90
- camera_x.append(pos[0])
91
- camera_y.append(pos[1])
92
- camera_z.append(pos[2])
93
-
94
- camera_x = np.array(camera_x)
95
- camera_y = np.array(camera_y)
96
- camera_z = np.array(camera_z)
97
-
98
- # Parse hand positions (world frame)
99
- hands_frames = hands_3d.get('frames', [])
100
- left_hand_x = np.array([f.get('left_hand', {}).get('position', [0,0,0])[0] for f in hands_frames] or [0]*num_frames)
101
- left_hand_y = np.array([f.get('left_hand', {}).get('position', [0,0,0])[1] for f in hands_frames] or [0]*num_frames)
102
- left_hand_z = np.array([f.get('left_hand', {}).get('position', [0,0,0])[2] for f in hands_frames] or [0]*num_frames)
103
-
104
- right_hand_x = np.array([f.get('right_hand', {}).get('position', [0,0,0])[0] for f in hands_frames] or [0]*num_frames)
105
- right_hand_y = np.array([f.get('right_hand', {}).get('position', [0,0,0])[1] for f in hands_frames] or [0]*num_frames)
106
- right_hand_z = np.array([f.get('right_hand', {}).get('position', [0,0,0])[2] for f in hands_frames] or [0]*num_frames)
107
-
108
- # Parse hand orientations
109
- ee_frames = end_effector.get('frames', [])
110
- left_hand_roll = np.array([f.get('left_hand', {}).get('orientation', [0,0,0])[0] for f in ee_frames] or [0]*num_frames)
111
- left_hand_pitch = np.array([f.get('left_hand', {}).get('orientation', [0,0,0])[1] for f in ee_frames] or [0]*num_frames)
112
- left_hand_yaw = np.array([f.get('left_hand', {}).get('orientation', [0,0,0])[2] for f in ee_frames] or [0]*num_frames)
113
-
114
- right_hand_roll = np.array([f.get('right_hand', {}).get('orientation', [0,0,0])[0] for f in ee_frames] or [0]*num_frames)
115
- right_hand_pitch = np.array([f.get('right_hand', {}).get('orientation', [0,0,0])[1] for f in ee_frames] or [0]*num_frames)
116
- right_hand_yaw = np.array([f.get('right_hand', {}).get('orientation', [0,0,0])[2] for f in ee_frames] or [0]*num_frames)
117
-
118
- # Parse 21 joint positions per hand
119
- left_hand_joints = np.array([
120
- f.get('left_hand', {}).get('landmarks_3d', np.zeros((21, 3)))
121
- for f in hands_frames
122
- ] or [np.zeros((21, 3))] * num_frames)
123
-
124
- right_hand_joints = np.array([
125
- f.get('right_hand', {}).get('landmarks_3d', np.zeros((21, 3)))
126
- for f in hands_frames
127
- ] or [np.zeros((21, 3))] * num_frames)
128
 
129
  return TrajectoryData(
130
- timestamps=timestamps,
131
- camera_x=camera_x, camera_y=camera_y, camera_z=camera_z,
132
- left_hand_x=left_hand_x, left_hand_y=left_hand_y, left_hand_z=left_hand_z,
133
- right_hand_x=right_hand_x, right_hand_y=right_hand_y, right_hand_z=right_hand_z,
134
- left_hand_roll=left_hand_roll, left_hand_pitch=left_hand_pitch, left_hand_yaw=left_hand_yaw,
135
- right_hand_roll=right_hand_roll, right_hand_pitch=right_hand_pitch, right_hand_yaw=right_hand_yaw,
136
- left_hand_joints=left_hand_joints,
137
- right_hand_joints=right_hand_joints
 
 
 
 
 
 
 
 
138
  )
139
 
140
 
141
- def create_trajectory_plots(data: TrajectoryData) -> go.Figure:
142
- """Create visualization with 15 plots (57 data streams total, 42 stored only)."""
143
-
144
  fig = make_subplots(
145
  rows=5, cols=3,
146
  subplot_titles=[
147
  'Camera X (m)', 'Camera Y (m)', 'Camera Z (m)',
148
- 'Left Hand X (m)', 'Left Hand Y (m)', 'Left Hand Z (m)',
149
- 'Right Hand X (m)', 'Right Hand Y (m)', 'Right Hand Z (m)',
150
- 'Left Hand Roll', 'Left Hand Pitch', 'Left Hand Yaw',
151
- 'Right Hand Roll', 'Right Hand Pitch', 'Right Hand Yaw',
152
  ],
153
- vertical_spacing=0.08,
154
- horizontal_spacing=0.05
155
  )
156
 
157
  t = data.timestamps
158
-
159
- # Row 1: Camera world frame (blue)
160
- fig.add_trace(go.Scatter(x=t, y=data.camera_x, name='cam_x', line=dict(color='#2563eb')), row=1, col=1)
161
- fig.add_trace(go.Scatter(x=t, y=data.camera_y, name='cam_y', line=dict(color='#2563eb')), row=1, col=2)
162
- fig.add_trace(go.Scatter(x=t, y=data.camera_z, name='cam_z', line=dict(color='#2563eb')), row=1, col=3)
163
-
164
- # Row 2: Left hand position (red)
165
- fig.add_trace(go.Scatter(x=t, y=data.left_hand_x, name='L_x', line=dict(color='#dc2626')), row=2, col=1)
166
- fig.add_trace(go.Scatter(x=t, y=data.left_hand_y, name='L_y', line=dict(color='#dc2626')), row=2, col=2)
167
- fig.add_trace(go.Scatter(x=t, y=data.left_hand_z, name='L_z', line=dict(color='#dc2626')), row=2, col=3)
168
-
169
- # Row 3: Right hand position (green)
170
- fig.add_trace(go.Scatter(x=t, y=data.right_hand_x, name='R_x', line=dict(color='#16a34a')), row=3, col=1)
171
- fig.add_trace(go.Scatter(x=t, y=data.right_hand_y, name='R_y', line=dict(color='#16a34a')), row=3, col=2)
172
- fig.add_trace(go.Scatter(x=t, y=data.right_hand_z, name='R_z', line=dict(color='#16a34a')), row=3, col=3)
173
-
174
- # Row 4: Left hand orientation (orange)
175
- fig.add_trace(go.Scatter(x=t, y=data.left_hand_roll, name='L_roll', line=dict(color='#ea580c')), row=4, col=1)
176
- fig.add_trace(go.Scatter(x=t, y=data.left_hand_pitch, name='L_pitch', line=dict(color='#ea580c')), row=4, col=2)
177
- fig.add_trace(go.Scatter(x=t, y=data.left_hand_yaw, name='L_yaw', line=dict(color='#ea580c')), row=4, col=3)
178
-
179
- # Row 5: Right hand orientation (purple)
180
- fig.add_trace(go.Scatter(x=t, y=data.right_hand_roll, name='R_roll', line=dict(color='#9333ea')), row=5, col=1)
181
- fig.add_trace(go.Scatter(x=t, y=data.right_hand_pitch, name='R_pitch', line=dict(color='#9333ea')), row=5, col=2)
182
- fig.add_trace(go.Scatter(x=t, y=data.right_hand_yaw, name='R_yaw', line=dict(color='#9333ea')), row=5, col=3)
183
-
184
- fig.update_layout(
185
- height=1200,
186
- title_text="Trajectory Data Visualization (15 plots, 57 data streams)",
187
- showlegend=False,
188
- template="plotly_white"
189
- )
190
-
191
  fig.update_xaxes(title_text="Time (sec)")
192
 
193
  return fig
194
 
195
 
196
- def create_3d_trajectory_plot(data: TrajectoryData) -> go.Figure:
197
- """Create 3D visualization of camera and hand trajectories."""
198
-
199
  fig = go.Figure()
200
 
201
- # Camera trajectory (blue)
202
  fig.add_trace(go.Scatter3d(
203
  x=data.camera_x, y=data.camera_y, z=data.camera_z,
204
- mode='lines',
205
- name='Camera',
206
- line=dict(color='#2563eb', width=4)
207
  ))
208
 
209
- # Hand positions are already in world frame
210
- left_world_x = data.left_hand_x
211
- left_world_y = data.left_hand_y
212
- left_world_z = data.left_hand_z
213
-
214
  fig.add_trace(go.Scatter3d(
215
- x=left_world_x, y=left_world_y, z=left_world_z,
216
- mode='lines',
217
- name='Left Hand',
218
- line=dict(color='#dc2626', width=4)
219
  ))
220
 
221
- # Right hand trajectory (green)
222
- right_world_x = data.right_hand_x
223
- right_world_y = data.right_hand_y
224
- right_world_z = data.right_hand_z
225
-
226
  fig.add_trace(go.Scatter3d(
227
- x=right_world_x, y=right_world_y, z=right_world_z,
228
- mode='lines',
229
- name='Right Hand',
230
- line=dict(color='#16a34a', width=4)
231
  ))
232
 
233
  fig.update_layout(
234
  title='3D Trajectory (World Frame)',
235
- scene=dict(
236
- xaxis_title='X (m)',
237
- yaxis_title='Y (m)',
238
- zaxis_title='Z (m)',
239
- aspectmode='data',
240
- bgcolor='#fafafa'
241
- ),
242
- height=700,
243
- template="plotly_white"
244
  )
245
 
246
  return fig
247
 
248
 
249
- def list_episodes() -> list:
250
- """List all episodes in the dataset."""
251
- try:
252
- files = list_repo_files(DATASET_REPO, repo_type="dataset")
253
- episodes = set()
254
- for f in files:
255
- parts = f.split('/')
256
- if len(parts) > 1 and parts[0].startswith('episode'):
257
- episodes.add(parts[0])
258
- if not episodes:
259
- # Try finding any folder
260
- for f in files:
261
- parts = f.split('/')
262
- if len(parts) > 1:
263
- episodes.add(parts[0])
264
- return sorted(list(episodes)) if episodes else ["No episodes found"]
265
- except Exception as e:
266
- return [f"Error listing: {str(e)}"]
267
-
268
-
269
- def load_and_visualize(episode_id: str):
270
- """Load episode data and create visualizations."""
271
- if not episode_id or episode_id.startswith("Error") or episode_id == "No episodes found":
272
- empty_fig = go.Figure()
273
- empty_fig.add_annotation(text="Select an episode to visualize", showarrow=False, font_size=20)
274
- return empty_fig, empty_fig, "Select an episode from the dropdown"
275
-
276
- try:
277
- episode_path = Path(f"/tmp/{episode_id}")
278
- episode_path.mkdir(parents=True, exist_ok=True)
279
-
280
- # Try downloading metadata.json
281
  try:
282
- hf_hub_download(
283
- repo_id=DATASET_REPO,
284
- filename=f"{episode_id}/extracted/metadata.json",
285
- local_dir="/tmp",
286
- repo_type="dataset"
287
- )
288
- except:
289
- hf_hub_download(
290
- repo_id=DATASET_REPO,
291
- filename=f"{episode_id}/metadata.json",
292
- local_dir="/tmp",
293
- repo_type="dataset"
294
- )
295
-
296
- # Try downloading hands_3d.json
297
- try:
298
- hf_hub_download(
299
- repo_id=DATASET_REPO,
300
- filename=f"{episode_id}/hands_3d.json",
301
- local_dir="/tmp",
302
- repo_type="dataset"
303
- )
304
- except:
305
- pass
306
-
307
- # Try downloading end_effector.json
308
- try:
309
- hf_hub_download(
310
- repo_id=DATASET_REPO,
311
- filename=f"{episode_id}/end_effector.json",
312
- local_dir="/tmp",
313
- repo_type="dataset"
314
- )
315
- except:
316
- pass
317
-
318
- # Load data
319
- data = load_trajectory_data(Path(f"/tmp/{episode_id}"))
320
-
321
- # Create plots
322
- trajectory_plot = create_trajectory_plots(data)
323
- plot_3d = create_3d_trajectory_plot(data)
324
-
325
- # Stats
326
- stats = f"""
327
- ## Episode: {episode_id}
328
 
329
  | Metric | Value |
330
  |--------|-------|
331
- | Duration | {data.timestamps[-1]:.2f} seconds |
332
  | Frames | {len(data.timestamps)} |
333
- | Data streams | 57 total |
334
- | Visualized | 15 streams |
335
- | Stored (joints) | 42 streams |
336
- """
337
-
338
- return trajectory_plot, plot_3d, stats
339
-
340
- except Exception as e:
341
- empty_fig = go.Figure()
342
- empty_fig.add_annotation(text=f"Error: {str(e)}", showarrow=False, font_size=14)
343
- return empty_fig, empty_fig, f"**Error:** {str(e)}"
344
 
345
 
346
- # Build Gradio interface
347
- with gr.Blocks(
348
- title="DI Trajectory Visualizer",
349
- theme=gr.themes.Soft(primary_hue="blue", secondary_hue="green")
350
- ) as demo:
351
- gr.Markdown("""
352
  # Dynamic Intelligence - Trajectory Visualizer
353
 
354
- Visualize humanoid robot training data: camera poses, hand positions, and orientations.
355
 
356
- ### Data Streams (57 total)
357
- | Category | Streams | Description |
358
- |----------|---------|-------------|
359
- | Camera Position | 3 | X, Y, Z in world frame (meters) |
360
- | Left Hand Position | 3 | X, Y, Z in world frame (meters) |
361
- | Right Hand Position | 3 | X, Y, Z in world frame (meters) |
362
- | Left Hand Orientation | 3 | Roll, Pitch, Yaw (degrees) |
363
- | Right Hand Orientation | 3 | Roll, Pitch, Yaw (degrees) |
364
- | Hand Joints (stored) | 42 | 21 joints x 2 hands x XYZ |
365
- """)
366
-
367
- with gr.Row():
368
- episode_dropdown = gr.Dropdown(
369
- label="Select Episode",
370
- choices=list_episodes(),
371
- interactive=True,
372
- scale=3
 
 
 
 
 
 
 
 
 
 
373
  )
374
- load_btn = gr.Button("Load & Visualize", variant="primary", scale=1)
375
-
376
- stats_output = gr.Markdown()
377
-
378
- with gr.Tabs():
379
- with gr.TabItem("Time Series (15 plots)"):
380
- trajectory_plot = gr.Plot(label="Trajectory Data")
381
 
382
- with gr.TabItem("3D View"):
383
- plot_3d = gr.Plot(label="3D Trajectory")
384
-
385
- load_btn.click(
386
- fn=load_and_visualize,
387
- inputs=[episode_dropdown],
388
- outputs=[trajectory_plot, plot_3d, stats_output]
389
- )
390
-
391
- gr.Markdown("""
392
- ---
393
- **Data Source:** [DynamicIntelligence/humanoid-robots-training-dataset](https://huggingface.co/datasets/DynamicIntelligence/humanoid-robots-training-dataset)
394
- """)
395
 
396
- if __name__ == "__main__":
397
  demo.launch()
 
 
 
1
  #!/usr/bin/env python3
2
  """
3
+ DI Trajectory Visualizer - Simple version without audio dependencies
 
4
  """
5
 
 
 
 
 
 
6
  import json
7
  import numpy as np
8
+ from pathlib import Path
9
  from dataclasses import dataclass
10
+ from typing import List
11
 
12
+ # Use basic imports only
13
+ try:
14
+ import gradio as gr
15
+ GRADIO_AVAILABLE = True
16
+ except ImportError:
17
+ GRADIO_AVAILABLE = False
18
 
19
+ import plotly.graph_objects as go
20
+ from plotly.subplots import make_subplots
21
 
22
  @dataclass
23
  class TrajectoryData:
24
+ """Container for trajectory data."""
25
  timestamps: np.ndarray
 
26
  camera_x: np.ndarray
27
  camera_y: np.ndarray
28
  camera_z: np.ndarray
 
29
  left_hand_x: np.ndarray
30
  left_hand_y: np.ndarray
31
  left_hand_z: np.ndarray
 
32
  right_hand_x: np.ndarray
33
  right_hand_y: np.ndarray
34
  right_hand_z: np.ndarray
 
35
  left_hand_roll: np.ndarray
36
  left_hand_pitch: np.ndarray
37
  left_hand_yaw: np.ndarray
 
38
  right_hand_roll: np.ndarray
39
  right_hand_pitch: np.ndarray
40
  right_hand_yaw: np.ndarray
 
 
 
41
 
42
 
43
+ def create_sample_data() -> TrajectoryData:
44
+ """Create sample trajectory data for demo."""
45
+ n = 300 # 10 seconds at 30fps
46
+ t = np.linspace(0, 10, n)
47
+
48
+ # Camera moving in a path
49
+ camera_x = np.sin(t * 0.5) * 0.5
50
+ camera_y = np.cos(t * 0.3) * 0.3
51
+ camera_z = t * 0.1
52
+
53
+ # Left hand
54
+ left_hand_x = camera_x + np.sin(t * 2) * 0.3 + 0.2
55
+ left_hand_y = camera_y + np.cos(t * 2) * 0.2 - 0.3
56
+ left_hand_z = camera_z + np.sin(t) * 0.1
57
+
58
+ # Right hand
59
+ right_hand_x = camera_x + np.sin(t * 2 + np.pi) * 0.3 - 0.2
60
+ right_hand_y = camera_y + np.cos(t * 2 + np.pi) * 0.2 - 0.3
61
+ right_hand_z = camera_z + np.cos(t) * 0.1
62
+
63
+ # Orientations
64
+ left_hand_roll = np.sin(t * 3) * 30
65
+ left_hand_pitch = np.cos(t * 2) * 20
66
+ left_hand_yaw = np.sin(t * 1.5) * 45
67
+
68
+ right_hand_roll = np.sin(t * 3 + 1) * 30
69
+ right_hand_pitch = np.cos(t * 2 + 1) * 20
70
+ right_hand_yaw = np.sin(t * 1.5 + 1) * 45
71
 
72
+ return TrajectoryData(
73
+ timestamps=t,
74
+ camera_x=camera_x, camera_y=camera_y, camera_z=camera_z,
75
+ left_hand_x=left_hand_x, left_hand_y=left_hand_y, left_hand_z=left_hand_z,
76
+ right_hand_x=right_hand_x, right_hand_y=right_hand_y, right_hand_z=right_hand_z,
77
+ left_hand_roll=left_hand_roll, left_hand_pitch=left_hand_pitch, left_hand_yaw=left_hand_yaw,
78
+ right_hand_roll=right_hand_roll, right_hand_pitch=right_hand_pitch, right_hand_yaw=right_hand_yaw
79
+ )
80
+
81
+
82
+ def load_from_json(episode_path: str) -> TrajectoryData:
83
+ """Load trajectory data from JSON files."""
84
+ path = Path(episode_path)
85
+
86
+ # Try to load metadata
87
+ metadata = {"poses": []}
88
+ for meta_file in ["metadata.json", "extracted/metadata.json"]:
89
+ meta_path = path / meta_file
90
+ if meta_path.exists():
91
+ with open(meta_path) as f:
92
+ metadata = json.load(f)
93
+ break
94
+
95
+ # Try to load hands_3d
96
  hands_3d = {"frames": []}
97
+ hands_path = path / "hands_3d.json"
98
+ if hands_path.exists():
99
+ with open(hands_path) as f:
100
  hands_3d = json.load(f)
101
 
102
+ # Try to load end_effector
 
103
  end_effector = {"frames": []}
104
+ ee_path = path / "end_effector.json"
105
+ if ee_path.exists():
106
+ with open(ee_path) as f:
107
  end_effector = json.load(f)
108
 
109
+ # Parse frames
110
  frames = metadata.get('frames', metadata.get('poses', []))
111
+ n = max(len(frames), 1)
112
  fps = metadata.get('fps', 30)
 
113
 
114
+ timestamps = np.arange(n) / fps
115
+
116
+ # Camera positions
117
  camera_x, camera_y, camera_z = [], [], []
118
  for f in frames:
119
+ pos = f.get('camera_pose', {}).get('position', f.get('position', [0, 0, 0]))
120
+ camera_x.append(pos[0] if len(pos) > 0 else 0)
121
+ camera_y.append(pos[1] if len(pos) > 1 else 0)
122
+ camera_z.append(pos[2] if len(pos) > 2 else 0)
123
+
124
+ # Hand positions
125
+ hframes = hands_3d.get('frames', [])
126
+ left_hand_x = [f.get('left_hand', {}).get('position', [0,0,0])[0] for f in hframes] or [0]*n
127
+ left_hand_y = [f.get('left_hand', {}).get('position', [0,0,0])[1] for f in hframes] or [0]*n
128
+ left_hand_z = [f.get('left_hand', {}).get('position', [0,0,0])[2] for f in hframes] or [0]*n
129
+ right_hand_x = [f.get('right_hand', {}).get('position', [0,0,0])[0] for f in hframes] or [0]*n
130
+ right_hand_y = [f.get('right_hand', {}).get('position', [0,0,0])[1] for f in hframes] or [0]*n
131
+ right_hand_z = [f.get('right_hand', {}).get('position', [0,0,0])[2] for f in hframes] or [0]*n
132
+
133
+ # Orientations
134
+ eframes = end_effector.get('frames', [])
135
+ left_hand_roll = [f.get('left_hand', {}).get('orientation', [0,0,0])[0] for f in eframes] or [0]*n
136
+ left_hand_pitch = [f.get('left_hand', {}).get('orientation', [0,0,0])[1] for f in eframes] or [0]*n
137
+ left_hand_yaw = [f.get('left_hand', {}).get('orientation', [0,0,0])[2] for f in eframes] or [0]*n
138
+ right_hand_roll = [f.get('right_hand', {}).get('orientation', [0,0,0])[0] for f in eframes] or [0]*n
139
+ right_hand_pitch = [f.get('right_hand', {}).get('orientation', [0,0,0])[1] for f in eframes] or [0]*n
140
+ right_hand_yaw = [f.get('right_hand', {}).get('orientation', [0,0,0])[2] for f in eframes] or [0]*n
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
 
142
  return TrajectoryData(
143
+ timestamps=np.array(timestamps) if len(timestamps) else np.array([0]),
144
+ camera_x=np.array(camera_x) if camera_x else np.array([0]),
145
+ camera_y=np.array(camera_y) if camera_y else np.array([0]),
146
+ camera_z=np.array(camera_z) if camera_z else np.array([0]),
147
+ left_hand_x=np.array(left_hand_x),
148
+ left_hand_y=np.array(left_hand_y),
149
+ left_hand_z=np.array(left_hand_z),
150
+ right_hand_x=np.array(right_hand_x),
151
+ right_hand_y=np.array(right_hand_y),
152
+ right_hand_z=np.array(right_hand_z),
153
+ left_hand_roll=np.array(left_hand_roll),
154
+ left_hand_pitch=np.array(left_hand_pitch),
155
+ left_hand_yaw=np.array(left_hand_yaw),
156
+ right_hand_roll=np.array(right_hand_roll),
157
+ right_hand_pitch=np.array(right_hand_pitch),
158
+ right_hand_yaw=np.array(right_hand_yaw)
159
  )
160
 
161
 
162
+ def create_time_series_plot(data: TrajectoryData) -> go.Figure:
163
+ """Create 15-subplot visualization."""
 
164
  fig = make_subplots(
165
  rows=5, cols=3,
166
  subplot_titles=[
167
  'Camera X (m)', 'Camera Y (m)', 'Camera Z (m)',
168
+ 'Left Hand X', 'Left Hand Y', 'Left Hand Z',
169
+ 'Right Hand X', 'Right Hand Y', 'Right Hand Z',
170
+ 'Left Roll', 'Left Pitch', 'Left Yaw',
171
+ 'Right Roll', 'Right Pitch', 'Right Yaw',
172
  ],
173
+ vertical_spacing=0.06,
174
+ horizontal_spacing=0.04
175
  )
176
 
177
  t = data.timestamps
178
+ colors = ['#2563eb', '#dc2626', '#16a34a', '#ea580c', '#9333ea']
179
+
180
+ # Row 1: Camera
181
+ fig.add_trace(go.Scatter(x=t, y=data.camera_x, line=dict(color=colors[0], width=1), showlegend=False), row=1, col=1)
182
+ fig.add_trace(go.Scatter(x=t, y=data.camera_y, line=dict(color=colors[0], width=1), showlegend=False), row=1, col=2)
183
+ fig.add_trace(go.Scatter(x=t, y=data.camera_z, line=dict(color=colors[0], width=1), showlegend=False), row=1, col=3)
184
+
185
+ # Row 2: Left hand position
186
+ fig.add_trace(go.Scatter(x=t, y=data.left_hand_x, line=dict(color=colors[1], width=1), showlegend=False), row=2, col=1)
187
+ fig.add_trace(go.Scatter(x=t, y=data.left_hand_y, line=dict(color=colors[1], width=1), showlegend=False), row=2, col=2)
188
+ fig.add_trace(go.Scatter(x=t, y=data.left_hand_z, line=dict(color=colors[1], width=1), showlegend=False), row=2, col=3)
189
+
190
+ # Row 3: Right hand position
191
+ fig.add_trace(go.Scatter(x=t, y=data.right_hand_x, line=dict(color=colors[2], width=1), showlegend=False), row=3, col=1)
192
+ fig.add_trace(go.Scatter(x=t, y=data.right_hand_y, line=dict(color=colors[2], width=1), showlegend=False), row=3, col=2)
193
+ fig.add_trace(go.Scatter(x=t, y=data.right_hand_z, line=dict(color=colors[2], width=1), showlegend=False), row=3, col=3)
194
+
195
+ # Row 4: Left hand orientation
196
+ fig.add_trace(go.Scatter(x=t, y=data.left_hand_roll, line=dict(color=colors[3], width=1), showlegend=False), row=4, col=1)
197
+ fig.add_trace(go.Scatter(x=t, y=data.left_hand_pitch, line=dict(color=colors[3], width=1), showlegend=False), row=4, col=2)
198
+ fig.add_trace(go.Scatter(x=t, y=data.left_hand_yaw, line=dict(color=colors[3], width=1), showlegend=False), row=4, col=3)
199
+
200
+ # Row 5: Right hand orientation
201
+ fig.add_trace(go.Scatter(x=t, y=data.right_hand_roll, line=dict(color=colors[4], width=1), showlegend=False), row=5, col=1)
202
+ fig.add_trace(go.Scatter(x=t, y=data.right_hand_pitch, line=dict(color=colors[4], width=1), showlegend=False), row=5, col=2)
203
+ fig.add_trace(go.Scatter(x=t, y=data.right_hand_yaw, line=dict(color=colors[4], width=1), showlegend=False), row=5, col=3)
204
+
205
+ fig.update_layout(height=1000, showlegend=False, title_text="57 Data Streams Visualization")
 
 
 
 
 
206
  fig.update_xaxes(title_text="Time (sec)")
207
 
208
  return fig
209
 
210
 
211
+ def create_3d_plot(data: TrajectoryData) -> go.Figure:
212
+ """Create 3D trajectory plot."""
 
213
  fig = go.Figure()
214
 
 
215
  fig.add_trace(go.Scatter3d(
216
  x=data.camera_x, y=data.camera_y, z=data.camera_z,
217
+ mode='lines', name='Camera', line=dict(color='#2563eb', width=4)
 
 
218
  ))
219
 
 
 
 
 
 
220
  fig.add_trace(go.Scatter3d(
221
+ x=data.left_hand_x, y=data.left_hand_y, z=data.left_hand_z,
222
+ mode='lines', name='Left Hand', line=dict(color='#dc2626', width=4)
 
 
223
  ))
224
 
 
 
 
 
 
225
  fig.add_trace(go.Scatter3d(
226
+ x=data.right_hand_x, y=data.right_hand_y, z=data.right_hand_z,
227
+ mode='lines', name='Right Hand', line=dict(color='#16a34a', width=4)
 
 
228
  ))
229
 
230
  fig.update_layout(
231
  title='3D Trajectory (World Frame)',
232
+ scene=dict(xaxis_title='X', yaxis_title='Y', zaxis_title='Z', aspectmode='data'),
233
+ height=600
 
 
 
 
 
 
 
234
  )
235
 
236
  return fig
237
 
238
 
239
+ def visualize(source: str):
240
+ """Main visualization function."""
241
+ if source == "Sample Data":
242
+ data = create_sample_data()
243
+ info = "Using generated sample data (10 seconds, 300 frames)"
244
+ else:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
  try:
246
+ data = load_from_json(source)
247
+ info = f"Loaded from: {source}"
248
+ except Exception as e:
249
+ data = create_sample_data()
250
+ info = f"Error loading data: {e}. Using sample data."
251
+
252
+ time_series = create_time_series_plot(data)
253
+ plot_3d = create_3d_plot(data)
254
+
255
+ stats = f"""
256
+ ## Data Statistics
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
257
 
258
  | Metric | Value |
259
  |--------|-------|
260
+ | Duration | {data.timestamps[-1]:.2f} sec |
261
  | Frames | {len(data.timestamps)} |
262
+ | Visualized Streams | 15 |
263
+ | Total Streams | 57 (including 42 joint positions) |
264
+
265
+ **Info:** {info}
266
+ """
267
+
268
+ return time_series, plot_3d, stats
 
 
 
 
269
 
270
 
271
+ # Gradio Interface
272
+ if GRADIO_AVAILABLE:
273
+ with gr.Blocks(title="DI Trajectory Visualizer") as demo:
274
+ gr.Markdown("""
 
 
275
  # Dynamic Intelligence - Trajectory Visualizer
276
 
277
+ Visualize 57 data streams from humanoid robot training data.
278
 
279
+ ### Data Streams
280
+ - **Camera**: X, Y, Z position (world frame)
281
+ - **Left Hand**: X, Y, Z position + Roll, Pitch, Yaw
282
+ - **Right Hand**: X, Y, Z position + Roll, Pitch, Yaw
283
+ - **Joints**: 21 keypoints × 2 hands × XYZ (stored, not visualized)
284
+ """)
285
+
286
+ with gr.Row():
287
+ source_input = gr.Dropdown(
288
+ choices=["Sample Data"],
289
+ value="Sample Data",
290
+ label="Data Source"
291
+ )
292
+ load_btn = gr.Button("Visualize", variant="primary")
293
+
294
+ stats_output = gr.Markdown()
295
+
296
+ with gr.Tabs():
297
+ with gr.TabItem("Time Series (15 plots)"):
298
+ time_plot = gr.Plot()
299
+ with gr.TabItem("3D View"):
300
+ plot_3d = gr.Plot()
301
+
302
+ load_btn.click(
303
+ fn=visualize,
304
+ inputs=[source_input],
305
+ outputs=[time_plot, plot_3d, stats_output]
306
  )
 
 
 
 
 
 
 
307
 
308
+ # Auto-load sample data
309
+ demo.load(
310
+ fn=lambda: visualize("Sample Data"),
311
+ outputs=[time_plot, plot_3d, stats_output]
312
+ )
 
 
 
 
 
 
 
 
313
 
 
314
  demo.launch()
315
+ else:
316
+ print("Gradio not available. Install with: pip install gradio")
requirements.txt CHANGED
@@ -1,5 +1,3 @@
1
- gradio==4.44.0
2
  plotly>=5.18.0
3
  numpy>=1.24.0
4
- huggingface_hub==0.21.4
5
- pandas>=2.0.0
 
 
1
  plotly>=5.18.0
2
  numpy>=1.24.0
3
+ gradio==4.31.0