This will make the black disc fly in toward the center from the tray’s corner as intended, with clean and controllable motion.

tray_sim.py

@@ -15,6 +15,19 @@ def run_tray_simulation(seed=0, num_objects=N_OBJECTS, azimuth=45, elevation=-25
     model = mujoco.MjModel.from_xml_path(MODEL_PATH)
     data = mujoco.MjData(model)
 
+    # Pusher setup
+    pusher_pos = np.array([0.35, -0.35, 0.08])  # Slightly outside tray corner
+    pusher_quat = [1, 0, 0, 0]
+    data.qpos[pusher_idx:pusher_idx+3] = pusher_pos
+    data.qpos[pusher_idx+3:pusher_idx+7] = pusher_quat
+    
+    # Compute velocity towards center
+    target = np.array([0.0, 0.0, 0.05])
+    direction = target - pusher_pos
+    direction[2] = 0.0
+    direction /= np.linalg.norm(direction)
+    push_velocity = 0.15 * direction  # Adjust magnitude
+
     # Object initialization
     for i in range(num_objects):
         obj_start = i * 7  # 7 DOF per free joint body
@@ -46,7 +59,8 @@ def run_tray_simulation(seed=0, num_objects=N_OBJECTS, azimuth=45, elevation=-25
     for t in range(SIM_STEPS):
         if t == PUSH_START_STEP:
             # Activate pusher
-            data.qvel[pusher_vel_idx:pusher_vel_idx+3] = [-0.05, 0.05, 0.0]  # toward center
+            #data.qvel[pusher_vel_idx:pusher_vel_idx+3] = [-0.05, 0.05, 0.0]  # toward center
+            data.qvel[pusher_vel_idx:pusher_vel_idx+3] = push_velocity  # toward center
 
         mujoco.mj_step(model, data)
         renderer.update_scene(data, camera=cam, scene_option=scene_option)