add: Pusher env

envs/__init__.py

@@ -108,8 +108,9 @@ from .mujoco import reacher_translator, reacher_policies
 from .mujoco import hopper_translator, hopper_policies
 from .mujoco import walker2d_translator, walker2d_policies
 
-from .mujoco import halfcheetah_translator, halfcheetah_policies
-from .mujoco import ant_translator, ant_policies
+
+
+
 
 REGISTRY["invertedPendulum_init_translator"] = invertedPendulum_translator.GameDescriber
 REGISTRY["invertedPendulum_basic_translator"] = invertedPendulum_translator.BasicStateSequenceTranslator
@@ -135,12 +136,17 @@ REGISTRY["walker2d_basic_translator"] = walker2d_translator.BasicStateSequenceTr
 REGISTRY["walker2d_policies"] = [walker2d_policies.pseudo_random_policy, walker2d_policies.real_random_policy]
 
 
-
+from .mujoco import halfcheetah_translator, halfcheetah_policies
 REGISTRY["halfcheetah_init_translator"] = halfcheetah_translator.GameDescriber
 REGISTRY["halfcheetah_basic_translator"] = halfcheetah_translator.BasicStateSequenceTranslator
 REGISTRY["halfcheetah_policies"] = [halfcheetah_policies.pseudo_random_policy, halfcheetah_policies.real_random_policy]
 
+from .mujoco import pusher_translator, pusher_policies
+REGISTRY["pusher_init_translator"] = pusher_translator.GameDescriber
+REGISTRY["pusher_basic_translator"] = pusher_translator.BasicStateSequenceTranslator
+REGISTRY["pusher_policies"] = [pusher_policies.pseudo_random_policy, pusher_policies.real_random_policy]
 
+from .mujoco import ant_translator, ant_policies
 REGISTRY["ant_init_translator"] = ant_translator.GameDescriber
 REGISTRY["ant_basic_translator"] = ant_translator.BasicStateSequenceTranslator
 REGISTRY["ant_policies"] = [ant_policies.pseudo_random_policy, ant_policies.real_random_policy]

envs/mujoco/ant_translator.py

@@ -1,4 +1,4 @@
-'''
+'''Ant
 Action Space Box(-1.0, 1.0, (8,), float32)
 Observation Space Box(-inf, inf, (27,), float64)
 '''

envs/mujoco/pusher_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [4 * random.random() - 2 for i in range(7)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [4 * random.random() - 2 for i in range(7)]

envs/mujoco/pusher_translator.py

@@ -0,0 +1,93 @@
+'''Pusher
+Action Space Box(-2.0, 2.0, (7,), float32)
+Observation Space Box(-inf, inf, (23,), float64)
+'''
+import math
+
+class BasicLevelTranslator:
+    def __init__(self):
+        pass
+
+    def translate(self, state):
+
+        joint_angles = state[:7]
+        joint_velocities = state[7:14]
+        fingertip_coords = state[14:17]
+        object_coords = state[17:20]
+        goal_coords = state[20:]
+
+        joint_angle_degrees = [math.degrees(angle) for angle in joint_angles]
+        joint_velocity_degrees = [math.degrees(velocity) for velocity in joint_velocities]
+
+        res = (f"Rotation of the panning shoulder: {joint_angle_degrees[0]:.2f} degrees, "
+               f"Rotation of the shoulder lifting joint: {joint_angle_degrees[1]:.2f} degrees, "
+               f"Rotation of the shoulder rolling joint: {joint_angle_degrees[2]:.2f} degrees, "
+               f"Rotation of the elbow joint: {joint_angle_degrees[3]:.2f} degrees, "
+               f"Rotation of the forearm rolling joint: {joint_angle_degrees[4]:.2f} degrees, "
+               f"Rotation of the wrist flexing joint: {joint_angle_degrees[5]:.2f} degrees, "
+               f"Rotation of the wrist rolling joint: {joint_angle_degrees[6]:.2f} degrees, "
+               f"Rotational velocity of the panning shoulder: {joint_velocity_degrees[0]:.2f} degrees/s, "
+               f"Rotational velocity of the shoulder lifting joint: {joint_velocity_degrees[1]:.2f} degrees/s, "
+               f"Rotational velocity of the shoulder rolling joint: {joint_velocity_degrees[2]:.2f} degrees/s, "
+               f"Rotational velocity of the elbow joint: {joint_velocity_degrees[3]:.2f} degrees/s, "
+               f"Rotational velocity of the forearm rolling joint: {joint_velocity_degrees[4]:.2f} degrees/s, "
+               f"Rotational velocity of the wrist flexing joint: {joint_velocity_degrees[5]:.2f} degrees/s, "
+               f"Rotational velocity of the wrist rolling joint: {joint_velocity_degrees[6]:.2f} degrees/s, "
+               f"Fingertip coordinates (x, y, z): ({fingertip_coords[0]:.2f}, {fingertip_coords[1]:.2f}, {fingertip_coords[2]:.2f}), "
+               f"Object coordinates (x, y, z): ({object_coords[0]:.2f}, {object_coords[1]:.2f}, {object_coords[2]:.2f}), "
+               f"Goal coordinates (x, y, z): ({goal_coords[0]:.2f}, {goal_coords[1]:.2f}, {goal_coords[2]:.2f}).")
+        return res
+
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+        }
+        self.reward_desc_dict = {
+        }
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len): 
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+
+    def describe_goal(self):
+        return "The goal is to move the target cylinder (object) to the goal position using the robot's end effector (fingertip)."
+
+    def describe_game(self):
+        return ("In the Pusher game, you control a multi-jointed robot arm to manipulate a target cylinder (object) "
+                "and place it in a goal position using the robot's fingertip (end effector). The robot has shoulder, elbow, "
+                "forearm, and wrist joints that you can control with torque values. The observation space includes joint angles, "
+                "angular velocities of joints, fingertip coordinates, object coordinates, and goal coordinates. The reward is "
+                "based on the distance between the fingertip and the object, the distance between the object and the goal, "
+                "and control penalties for large actions.")
+
+    def describe_action(self):
+        return ("Your next move: \n Please provide a list of 7 numerical values within the range [-2, 2], "
+                "representing the torques applied to the robot's joints (shoulder, elbow, forearm, and wrist).")
+
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for info in infos:
+            assert 'state' in info, "info should contain state information"
+
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = ("Take Action: Apply Torques - "
+                           "Shoulder Pan: {:.2f}, Shoulder Lift: {:.2f}, Shoulder Roll: {:.2f}, "
+                           "Elbow Flex: {:.2f}, Forearm Roll: {:.2f}, Wrist Flex: {:.2f}, Wrist Roll: {:.2f}"
+                           ).format(info['action'][0], info['action'][1], info['action'][2], info['action'][3],
+                                    info['action'][4], info['action'][5], info['action'][6])
+
+            reward_desc = f"Result: Reward of {info['reward']:.2f}"
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(f"{state_desc}\n{action_desc}\n{reward_desc}\nTransit to {next_state_desc}")
+        return descriptions
+

record_reflexion.csv

@@ -16,4 +16,5 @@ Hopper-v4,1,expert,3542.2
 Walker2d-v4,1,expert,5000.0
 Swimmer-v4,1,expert,44.4
 Reacher-v4,1,expert,-2.6
+Pusher-v4,1,expert,-52.3
 

shell/test_mujoco_hopper.sh

@@ -1,5 +1,17 @@
 # Hopper-v4
 
+# COT
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+
 # REFLEXION
 python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
 

shell/test_mujoco_invertedPendulum.sh

@@ -1,14 +1,14 @@
 # InvertedPendulum-v4
 
 # COT
-python main_reflexion.py --env_name InvertedPendulum-v1 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
 
-python main_reflexion.py --env_name InvertedPendulum-v1 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
 
 # SPP
-python main_reflexion.py --env_name InvertedPendulum-v1 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
 
-python main_reflexion.py --env_name InvertedPendulum-v1 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
 
 
 

shell/test_mujoco_pusher.sh

@@ -0,0 +1,27 @@
+# Pusher-v4
+
+# COT
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+# REFLEXION
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator