Refactor environment configuration and mujoco_server.py for improved Nova API integration

- Removed the NOVA_USE_STATE_STREAM variable from .env.example as it is no longer needed for digital twin mode.
- Introduced a new .env.off.local file for comprehensive Nova API configuration, including instance URL, access token, and robot settings.
- Enhanced mujoco_server.py to support bidirectional control and state synchronization with the Nova API, allowing for real-time interaction while maintaining physics simulation.
- Updated UR5 environment to manage joint positions from Nova, ensuring accurate control and interaction with simulated objects.

.env.example

@@ -25,8 +25,3 @@ NOVA_TCP_NAME=Flange
 
 # Optional: Response rate in milliseconds (default: 200)
 NOVA_RESPONSE_RATE_MS=200
-
-# Optional: Enable digital twin mode (mirror real robot state)
-# When true: simulation displays real robot pose (no physics simulation)
-# When false (default): local simulation runs with Nova jogging control
-NOVA_USE_STATE_STREAM=false

mujoco_server.py

@@ -170,21 +170,15 @@ def init_ur5(scene_name="scene"):
     # Check if Nova API is configured via environment variables
     nova_config = None
     if os.getenv('NOVA_INSTANCE_URL') and os.getenv('NOVA_ACCESS_TOKEN'):
-        # Nova API is configured - enable jogging for real robot control
-        # State streaming is DISABLED by default to allow local simulation to run
-        # (enables physics simulation and T-shape interaction)
-        # To enable digital twin mode, set NOVA_USE_STATE_STREAM=true in environment
-        use_state_stream = os.getenv('NOVA_USE_STATE_STREAM', 'false').lower() == 'true'
-
-        if use_state_stream:
-            print("Nova API: Digital Twin Mode - mirroring real robot state (no local physics)")
-        else:
-            print("Nova API: Jogging Mode - local simulation with Nova jog control")
-
+        # Nova API is configured - attempt bidirectional control
+        # State streaming: Mirror real robot pose while still running physics for objects (T-shape)
+        # Jogging: UI can send directional jog commands to control the real robot
+        # Note: If Nova connection fails, UR5Env will automatically fall back to internal IK
+        print("Nova API credentials detected - attempting bidirectional control")
         nova_config = {
-            "use_state_stream": use_state_stream,
-            "use_ik": False,  # Disabled: using jogging API instead
-            "use_jogging": True  # Enable bidirectional robot control
+            "use_state_stream": True,  # Attempt to mirror real robot pose
+            "use_ik": False,  # Use jogging API instead of Nova IK (falls back to internal IK if connection fails)
+            "use_jogging": True  # Attempt bidirectional robot control
         }
 
     if scene_name == "scene_t_push":
@@ -1041,11 +1035,6 @@ def index():
                             <span id="nova_mode_text">Digital Twin Mode</span> - Read Only
                         </div>
                     </div>
-                    <strong style="font-size: 0.9em;">Controllers:</strong><br>
-                    <span style="font-size: 0.85em;">
-                        State: <span id="nova_state_controller" style="color: var(--wb-highlight);">Internal</span><br>
-                        IK: <span id="nova_ik_controller" style="color: var(--wb-highlight);">Internal</span>
-                    </span>
                 </div>
             </div>
         </div>

robots/ur5/ur5_env.py

@@ -139,6 +139,7 @@ class UR5Env(gym.Env):
         self._use_nova_ik = False
         self._use_nova_jogging = False
         self._last_nova_sequence = None
+        self._latest_nova_joints = None
 
         # IK controller for end-effector control
         self.controller = IKController(
@@ -561,8 +562,17 @@ class UR5Env(gym.Env):
     def step_with_controller(self, dt: float = 0.002):
         """Step using controller (IK or direct joint control)."""
         if self._use_nova_state_stream:
-            # In digital twin mode, always sync from Nova (don't do local control)
+            # In digital twin mode, sync robot pose from Nova but still run physics
+            # This allows simulated objects (like T-shape) to interact with the real robot
             self._sync_nova_state()
+
+            # Set robot joint positions from Nova (override qpos)
+            if hasattr(self, '_latest_nova_joints') and self._latest_nova_joints is not None:
+                self.data.qpos[:6] = self._latest_nova_joints
+
+            # Still run physics step for other objects (T-shape, etc.)
+            mujoco.mj_step(self.model, self.data)
+            self.steps += 1
             return self._get_obs()
 
         if self._control_mode == 'ik':
@@ -645,7 +655,9 @@ class UR5Env(gym.Env):
         if not joint_positions or len(joint_positions) < 6:
             return False
 
-        self.data.qpos[:6] = np.array(joint_positions[:6], dtype=np.float32)
+        # Store latest joints for use in step_with_controller
+        self._latest_nova_joints = np.array(joint_positions[:6], dtype=np.float32)
+        self.data.qpos[:6] = self._latest_nova_joints
         self.data.qvel[:6] = 0.0
         self.data.ctrl[:6] = self.data.qpos[:6]
         mujoco.mj_forward(self.model, self.data)