robot_sim.py

import pybullet as p
import pybullet_data
import numpy as np
import matplotlib.pyplot as plt
import tempfile
import time

# Setup simulation
p.connect(p.DIRECT)
p.setAdditionalSearchPath(pybullet_data.getDataPath())
p.setGravity(0, 0, -9.8)

plane_id = p.loadURDF("plane.urdf")
robot_id = p.loadURDF("franka_panda/panda.urdf", basePosition=[0, 0, 0], useFixedBase=True)
cube_id = p.loadURDF("cube_small.urdf", basePosition=[0.6, 0, 0.02])
p.changeVisualShape(cube_id, -1, rgbaColor=[0, 0, 0, 1])

def get_joint_indices(robot):
    arm, fingers = [], []
    for i in range(p.getNumJoints(robot)):
        info = p.getJointInfo(robot, i)
        name = info[1].decode("utf-8")
        joint_type = info[2]
        if "finger" in name:
            fingers.append(i)
        elif joint_type == p.JOINT_REVOLUTE:
            arm.append(i)
    return arm, fingers

arm_joints, finger_joints = get_joint_indices(robot_id)
trajectory = []

def capture_view():
    width, height = 640, 640
    view_matrix = p.computeViewMatrix(cameraEyePosition=[1.2, 0, 1],
                                      cameraTargetPosition=[0, 0, 0.4],
                                      cameraUpVector=[0, 0, 1])
    proj_matrix = p.computeProjectionMatrixFOV(fov=60, aspect=1.0, nearVal=0.1, farVal=3.1)
    _, _, px, _, _ = p.getCameraImage(width=width, height=height,
                                      viewMatrix=view_matrix,
                                      projectionMatrix=proj_matrix)
    rgb_array = np.reshape(px, (height, width, 4))[:, :, :3]
    fig, ax = plt.subplots()
    ax.imshow(rgb_array.astype(np.uint8))
    ax.axis("off")
    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
    plt.savefig(tmp.name, bbox_inches='tight')
    plt.close()
    return tmp.name

def move_robot(joint_angles, grip=0.04, steps=100):
    if len(joint_angles) != 7:
        return "❌ Invalid joint input."

    for step in range(steps):
        blend = [(1 - step / steps) * p.getJointState(robot_id, j)[0] + (step / steps) * t
                 for j, t in zip(arm_joints, joint_angles)]
        for j, val in zip(arm_joints, blend):
            p.setJointMotorControl2(robot_id, j, p.POSITION_CONTROL, targetPosition=val)
        for f in finger_joints:
            p.setJointMotorControl2(robot_id, f, p.POSITION_CONTROL, targetPosition=grip)
        p.stepSimulation()
        pos = p.getLinkState(robot_id, arm_joints[-1])[0]
        trajectory.append(pos)
        time.sleep(0.005)

def perform_pick_and_place():
    approach = [0.0, -0.5, 0.3, -1.5, 0.0, 1.4, 0.6]
    pick = [0.0, -0.4, 0.3, -1.4, 0.0, 1.2, 0.6]
    place = [0.5, -0.4, 0.2, -1.8, 0.0, 1.4, 0.6]

    move_robot(approach)
    move_robot(pick, grip=0.0)  # Close gripper
    move_robot([a + 0.1 for a in pick], grip=0.0)
    move_robot(place, grip=0.0)
    move_robot(place, grip=0.04)  # Open gripper

def plot_trajectory():
    if not trajectory:
        return None
    traj = np.array(trajectory)
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot(traj[:, 0], traj[:, 1], traj[:, 2], label="End Effector Path")
    ax.scatter(traj[-1, 0], traj[-1, 1], traj[-1, 2], color='red', label='Final Position')
    ax.set_title("3D Trajectory")
    ax.legend()
    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
    plt.savefig(tmp.name)
    plt.close()
    return tmp.name

def robot_chatbot(user_input: str):
    user_input = user_input.lower().strip()
    if "pick" in user_input:
        trajectory.clear()
        perform_pick_and_place()
        sim_img = capture_view()
        traj_img = plot_trajectory()
        return "✅ Pick and place executed.", (sim_img, traj_img)

    try:
        values = [float(v.strip()) for v in user_input.split(",")]
        if len(values) == 7:
            trajectory.clear()
            move_robot(values)
            sim_img = capture_view()
            traj_img = plot_trajectory()
            return "✅ Moved to specified joint angles.", (sim_img, traj_img)
    except Exception:
        pass

    return "🤖 Command not recognized. Try 'pick', 'place', or provide 7 joint angles.", (capture_view(), None)