Spaces:
Runtime error
Runtime error
remove unused codes
Browse files- app.py +8 -11
- visualization/et_visualizer.py +3 -19
- visualization/logger.py +3 -6
- visualization/visualizer.py +0 -2
app.py
CHANGED
|
@@ -3,7 +3,6 @@ from gradio_rerun import Rerun
|
|
| 3 |
from data.loader import load_simulation_data
|
| 4 |
from visualization.visualizer import visualize_simulation
|
| 5 |
from visualization.et_visualizer import visualize_et_data
|
| 6 |
-
from pathlib import Path
|
| 7 |
|
| 8 |
|
| 9 |
def update_simulation_dropdown(file):
|
|
@@ -18,7 +17,6 @@ def update_simulation_dropdown(file):
|
|
| 18 |
def create_app():
|
| 19 |
with gr.Blocks() as demo:
|
| 20 |
with gr.Tabs() as tabs:
|
| 21 |
-
# Camera Simulation Tab
|
| 22 |
with gr.Tab("Camera Simulation"):
|
| 23 |
gr.Markdown("""
|
| 24 |
# Camera Simulation Visualizer
|
|
@@ -52,7 +50,6 @@ def create_app():
|
|
| 52 |
outputs=[viewer]
|
| 53 |
)
|
| 54 |
|
| 55 |
-
# E.T. Dataset Tab
|
| 56 |
with gr.Tab("E.T. Dataset"):
|
| 57 |
gr.Markdown("""
|
| 58 |
# E.T. Dataset Visualizer
|
|
@@ -69,14 +66,6 @@ def create_app():
|
|
| 69 |
file_types=[".npy"]
|
| 70 |
)
|
| 71 |
|
| 72 |
-
with gr.Row():
|
| 73 |
-
et_viewer = Rerun(streaming=False)
|
| 74 |
-
|
| 75 |
-
def process_et_files(traj_file, char_file):
|
| 76 |
-
if traj_file is None or char_file is None:
|
| 77 |
-
return None
|
| 78 |
-
return visualize_et_data(traj_file.name, char_file.name)
|
| 79 |
-
|
| 80 |
with gr.Row():
|
| 81 |
visualize_btn = gr.Button("Visualize")
|
| 82 |
visualize_btn.click(
|
|
@@ -85,6 +74,14 @@ def create_app():
|
|
| 85 |
outputs=[et_viewer]
|
| 86 |
)
|
| 87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
return demo
|
| 89 |
|
| 90 |
|
|
|
|
| 3 |
from data.loader import load_simulation_data
|
| 4 |
from visualization.visualizer import visualize_simulation
|
| 5 |
from visualization.et_visualizer import visualize_et_data
|
|
|
|
| 6 |
|
| 7 |
|
| 8 |
def update_simulation_dropdown(file):
|
|
|
|
| 17 |
def create_app():
|
| 18 |
with gr.Blocks() as demo:
|
| 19 |
with gr.Tabs() as tabs:
|
|
|
|
| 20 |
with gr.Tab("Camera Simulation"):
|
| 21 |
gr.Markdown("""
|
| 22 |
# Camera Simulation Visualizer
|
|
|
|
| 50 |
outputs=[viewer]
|
| 51 |
)
|
| 52 |
|
|
|
|
| 53 |
with gr.Tab("E.T. Dataset"):
|
| 54 |
gr.Markdown("""
|
| 55 |
# E.T. Dataset Visualizer
|
|
|
|
| 66 |
file_types=[".npy"]
|
| 67 |
)
|
| 68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
with gr.Row():
|
| 70 |
visualize_btn = gr.Button("Visualize")
|
| 71 |
visualize_btn.click(
|
|
|
|
| 74 |
outputs=[et_viewer]
|
| 75 |
)
|
| 76 |
|
| 77 |
+
with gr.Row():
|
| 78 |
+
et_viewer = Rerun(streaming=False)
|
| 79 |
+
|
| 80 |
+
def process_et_files(traj_file, char_file):
|
| 81 |
+
if traj_file is None or char_file is None:
|
| 82 |
+
return None
|
| 83 |
+
return visualize_et_data(traj_file.name, char_file.name)
|
| 84 |
+
|
| 85 |
return demo
|
| 86 |
|
| 87 |
|
visualization/et_visualizer.py
CHANGED
|
@@ -20,21 +20,17 @@ def load_trajectory_data(traj_file: str, char_file: str, num_cams: int = 30) ->
|
|
| 20 |
raw_trans = torch.clone(matrix_trajectory[:, :3, 3])
|
| 21 |
raw_rot = matrix_trajectory[:, :3, :3]
|
| 22 |
|
| 23 |
-
# Convert to 6D rotation representation
|
| 24 |
rot6d = raw_rot[:, :, :2].permute(0, 2, 1).reshape(-1, 6)
|
| 25 |
trajectory_feature = torch.hstack([rot6d, raw_trans]).permute(1, 0)
|
| 26 |
|
| 27 |
-
# Pad trajectory features
|
| 28 |
padded_trajectory_feature = F.pad(
|
| 29 |
trajectory_feature,
|
| 30 |
(0, num_cams - trajectory_feature.shape[1])
|
| 31 |
)
|
| 32 |
|
| 33 |
-
# Create padding mask
|
| 34 |
padding_mask = torch.ones((num_cams))
|
| 35 |
padding_mask[trajectory_feature.shape[1]:] = 0
|
| 36 |
|
| 37 |
-
# Load and pad character features
|
| 38 |
char_feature = torch.from_numpy(np.load(char_file)).to(torch.float32)
|
| 39 |
padding_size = num_cams - char_feature.shape[0]
|
| 40 |
padded_char_feature = F.pad(
|
|
@@ -55,7 +51,7 @@ class ETLogger(SimulationLogger):
|
|
| 55 |
super().__init__()
|
| 56 |
rr.init("et_visualization")
|
| 57 |
rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, timeless=True)
|
| 58 |
-
|
| 59 |
self.K = np.array([
|
| 60 |
[500, 0, 320],
|
| 61 |
[0, 500, 240],
|
|
@@ -63,11 +59,9 @@ class ETLogger(SimulationLogger):
|
|
| 63 |
])
|
| 64 |
|
| 65 |
def log_trajectory(self, trajectory: np.ndarray, padding_mask: np.ndarray):
|
| 66 |
-
"""Log camera trajectory."""
|
| 67 |
valid_frames = int(padding_mask.sum())
|
| 68 |
valid_trajectory = trajectory[:valid_frames]
|
| 69 |
|
| 70 |
-
# Log trajectory points
|
| 71 |
positions = valid_trajectory[:, :3, 3]
|
| 72 |
rr.log(
|
| 73 |
"world/trajectory/points",
|
|
@@ -78,7 +72,6 @@ class ETLogger(SimulationLogger):
|
|
| 78 |
timeless=True
|
| 79 |
)
|
| 80 |
|
| 81 |
-
# Log trajectory line
|
| 82 |
if len(positions) > 1:
|
| 83 |
lines = np.stack([positions[:-1], positions[1:]], axis=1)
|
| 84 |
rr.log(
|
|
@@ -90,17 +83,14 @@ class ETLogger(SimulationLogger):
|
|
| 90 |
timeless=True
|
| 91 |
)
|
| 92 |
|
| 93 |
-
# Log cameras
|
| 94 |
for k in range(valid_frames):
|
| 95 |
-
|
| 96 |
rr.set_time_sequence("frame_idx", k)
|
| 97 |
|
| 98 |
-
# Get camera pose
|
| 99 |
translation = valid_trajectory[k, :3, 3]
|
| 100 |
rotation_q = Rotation.from_matrix(
|
| 101 |
valid_trajectory[k, :3, :3]).as_quat()
|
| 102 |
|
| 103 |
-
# Log camera transform
|
| 104 |
rr.log(
|
| 105 |
f"world/camera",
|
| 106 |
rr.Transform3D(
|
|
@@ -109,7 +99,6 @@ class ETLogger(SimulationLogger):
|
|
| 109 |
),
|
| 110 |
)
|
| 111 |
|
| 112 |
-
# Log camera frustum
|
| 113 |
rr.log(
|
| 114 |
f"world/camera/image",
|
| 115 |
rr.Pinhole(
|
|
@@ -120,7 +109,6 @@ class ETLogger(SimulationLogger):
|
|
| 120 |
)
|
| 121 |
|
| 122 |
def log_character(self, char_feature: np.ndarray, padding_mask: np.ndarray):
|
| 123 |
-
"""Log character feature visualization."""
|
| 124 |
valid_frames = int(padding_mask.sum())
|
| 125 |
valid_char = char_feature[:, :valid_frames]
|
| 126 |
|
|
@@ -138,16 +126,13 @@ class ETLogger(SimulationLogger):
|
|
| 138 |
|
| 139 |
@spaces.GPU
|
| 140 |
def visualize_et_data(traj_file: str, char_file: str) -> Optional[str]:
|
| 141 |
-
"""Visualize E.T. dataset using Rerun."""
|
| 142 |
try:
|
| 143 |
-
|
| 144 |
data = load_trajectory_data(traj_file, char_file)
|
| 145 |
|
| 146 |
-
# Create temporary file for RRD
|
| 147 |
temp_dir = tempfile.mkdtemp()
|
| 148 |
rrd_path = os.path.join(temp_dir, "et_visualization.rrd")
|
| 149 |
|
| 150 |
-
# Initialize logger and log data
|
| 151 |
logger = ETLogger()
|
| 152 |
logger.log_trajectory(
|
| 153 |
data["raw_matrix_trajectory"].numpy(),
|
|
@@ -158,7 +143,6 @@ def visualize_et_data(traj_file: str, char_file: str) -> Optional[str]:
|
|
| 158 |
data["padding_mask"].numpy()
|
| 159 |
)
|
| 160 |
|
| 161 |
-
# Save visualization
|
| 162 |
rr.save(rrd_path)
|
| 163 |
return rrd_path
|
| 164 |
|
|
|
|
| 20 |
raw_trans = torch.clone(matrix_trajectory[:, :3, 3])
|
| 21 |
raw_rot = matrix_trajectory[:, :3, :3]
|
| 22 |
|
|
|
|
| 23 |
rot6d = raw_rot[:, :, :2].permute(0, 2, 1).reshape(-1, 6)
|
| 24 |
trajectory_feature = torch.hstack([rot6d, raw_trans]).permute(1, 0)
|
| 25 |
|
|
|
|
| 26 |
padded_trajectory_feature = F.pad(
|
| 27 |
trajectory_feature,
|
| 28 |
(0, num_cams - trajectory_feature.shape[1])
|
| 29 |
)
|
| 30 |
|
|
|
|
| 31 |
padding_mask = torch.ones((num_cams))
|
| 32 |
padding_mask[trajectory_feature.shape[1]:] = 0
|
| 33 |
|
|
|
|
| 34 |
char_feature = torch.from_numpy(np.load(char_file)).to(torch.float32)
|
| 35 |
padding_size = num_cams - char_feature.shape[0]
|
| 36 |
padded_char_feature = F.pad(
|
|
|
|
| 51 |
super().__init__()
|
| 52 |
rr.init("et_visualization")
|
| 53 |
rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, timeless=True)
|
| 54 |
+
|
| 55 |
self.K = np.array([
|
| 56 |
[500, 0, 320],
|
| 57 |
[0, 500, 240],
|
|
|
|
| 59 |
])
|
| 60 |
|
| 61 |
def log_trajectory(self, trajectory: np.ndarray, padding_mask: np.ndarray):
|
|
|
|
| 62 |
valid_frames = int(padding_mask.sum())
|
| 63 |
valid_trajectory = trajectory[:valid_frames]
|
| 64 |
|
|
|
|
| 65 |
positions = valid_trajectory[:, :3, 3]
|
| 66 |
rr.log(
|
| 67 |
"world/trajectory/points",
|
|
|
|
| 72 |
timeless=True
|
| 73 |
)
|
| 74 |
|
|
|
|
| 75 |
if len(positions) > 1:
|
| 76 |
lines = np.stack([positions[:-1], positions[1:]], axis=1)
|
| 77 |
rr.log(
|
|
|
|
| 83 |
timeless=True
|
| 84 |
)
|
| 85 |
|
|
|
|
| 86 |
for k in range(valid_frames):
|
| 87 |
+
|
| 88 |
rr.set_time_sequence("frame_idx", k)
|
| 89 |
|
|
|
|
| 90 |
translation = valid_trajectory[k, :3, 3]
|
| 91 |
rotation_q = Rotation.from_matrix(
|
| 92 |
valid_trajectory[k, :3, :3]).as_quat()
|
| 93 |
|
|
|
|
| 94 |
rr.log(
|
| 95 |
f"world/camera",
|
| 96 |
rr.Transform3D(
|
|
|
|
| 99 |
),
|
| 100 |
)
|
| 101 |
|
|
|
|
| 102 |
rr.log(
|
| 103 |
f"world/camera/image",
|
| 104 |
rr.Pinhole(
|
|
|
|
| 109 |
)
|
| 110 |
|
| 111 |
def log_character(self, char_feature: np.ndarray, padding_mask: np.ndarray):
|
|
|
|
| 112 |
valid_frames = int(padding_mask.sum())
|
| 113 |
valid_char = char_feature[:, :valid_frames]
|
| 114 |
|
|
|
|
| 126 |
|
| 127 |
@spaces.GPU
|
| 128 |
def visualize_et_data(traj_file: str, char_file: str) -> Optional[str]:
|
|
|
|
| 129 |
try:
|
| 130 |
+
|
| 131 |
data = load_trajectory_data(traj_file, char_file)
|
| 132 |
|
|
|
|
| 133 |
temp_dir = tempfile.mkdtemp()
|
| 134 |
rrd_path = os.path.join(temp_dir, "et_visualization.rrd")
|
| 135 |
|
|
|
|
| 136 |
logger = ETLogger()
|
| 137 |
logger.log_trajectory(
|
| 138 |
data["raw_matrix_trajectory"].numpy(),
|
|
|
|
| 143 |
data["padding_mask"].numpy()
|
| 144 |
)
|
| 145 |
|
|
|
|
| 146 |
rr.save(rrd_path)
|
| 147 |
return rrd_path
|
| 148 |
|
visualization/logger.py
CHANGED
|
@@ -18,10 +18,10 @@ class SimulationLogger:
|
|
| 18 |
def __init__(self):
|
| 19 |
rr.init("camera_simulation")
|
| 20 |
rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, timeless=True)
|
| 21 |
-
|
| 22 |
self.K = np.array([
|
| 23 |
-
[500, 0, 960],
|
| 24 |
-
[0, 500, 540],
|
| 25 |
[0, 0, 1]
|
| 26 |
])
|
| 27 |
|
|
@@ -93,7 +93,6 @@ class SimulationLogger:
|
|
| 93 |
timeless=True
|
| 94 |
)
|
| 95 |
|
| 96 |
-
# Add trajectory line
|
| 97 |
if len(camera_positions) > 1:
|
| 98 |
lines = np.stack(
|
| 99 |
[camera_positions[:-1], camera_positions[1:]], axis=1)
|
|
@@ -120,7 +119,6 @@ class SimulationLogger:
|
|
| 120 |
position = vector3_to_numpy(camera_frame['position'])
|
| 121 |
rotation_q = euler_to_quaternion(camera_frame['angle'])
|
| 122 |
|
| 123 |
-
# Log camera transform
|
| 124 |
rr.log(
|
| 125 |
"world/camera",
|
| 126 |
rr.Transform3D(
|
|
@@ -129,7 +127,6 @@ class SimulationLogger:
|
|
| 129 |
)
|
| 130 |
)
|
| 131 |
|
| 132 |
-
# Log camera image plane with intrinsics
|
| 133 |
rr.log(
|
| 134 |
"world/camera/image",
|
| 135 |
rr.Pinhole(
|
|
|
|
| 18 |
def __init__(self):
|
| 19 |
rr.init("camera_simulation")
|
| 20 |
rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, timeless=True)
|
| 21 |
+
|
| 22 |
self.K = np.array([
|
| 23 |
+
[500, 0, 960],
|
| 24 |
+
[0, 500, 540],
|
| 25 |
[0, 0, 1]
|
| 26 |
])
|
| 27 |
|
|
|
|
| 93 |
timeless=True
|
| 94 |
)
|
| 95 |
|
|
|
|
| 96 |
if len(camera_positions) > 1:
|
| 97 |
lines = np.stack(
|
| 98 |
[camera_positions[:-1], camera_positions[1:]], axis=1)
|
|
|
|
| 119 |
position = vector3_to_numpy(camera_frame['position'])
|
| 120 |
rotation_q = euler_to_quaternion(camera_frame['angle'])
|
| 121 |
|
|
|
|
| 122 |
rr.log(
|
| 123 |
"world/camera",
|
| 124 |
rr.Transform3D(
|
|
|
|
| 127 |
)
|
| 128 |
)
|
| 129 |
|
|
|
|
| 130 |
rr.log(
|
| 131 |
"world/camera/image",
|
| 132 |
rr.Pinhole(
|
visualization/visualizer.py
CHANGED
|
@@ -18,11 +18,9 @@ def visualize_simulation(file, simulation_index: Optional[int]) -> Optional[str]
|
|
| 18 |
print(f"Invalid simulation data or index: {simulation_index}")
|
| 19 |
return None
|
| 20 |
|
| 21 |
-
# Create temporary file for RRD
|
| 22 |
temp_dir = tempfile.mkdtemp()
|
| 23 |
rrd_path = os.path.join(temp_dir, "simulation.rrd")
|
| 24 |
|
| 25 |
-
# Log selected simulation
|
| 26 |
simulation = simulations[simulation_index]
|
| 27 |
|
| 28 |
logger = SimulationLogger()
|
|
|
|
| 18 |
print(f"Invalid simulation data or index: {simulation_index}")
|
| 19 |
return None
|
| 20 |
|
|
|
|
| 21 |
temp_dir = tempfile.mkdtemp()
|
| 22 |
rrd_path = os.path.join(temp_dir, "simulation.rrd")
|
| 23 |
|
|
|
|
| 24 |
simulation = simulations[simulation_index]
|
| 25 |
|
| 26 |
logger = SimulationLogger()
|