add reacher env and all mujoco envs now support COT, SPP, SELF-REFLEXION, EXE methods under L1&L3 setting.

envs/__init__.py

@@ -100,6 +100,11 @@ REGISTRY["RepresentedPong_basic_policies"] = [
 
 from .mujoco import invertedPendulum_translator, invertedPendulum_policies
 from .mujoco import invertedDoublePendulum_translator, invertedDoublePendulum_policies
+
+from .mujoco import swimmer_translator, swimmer_policies
+
+from .mujoco import reacher_translator, reacher_policies
+
 from .mujoco import hopper_translator, hopper_policies
 from .mujoco import walker2d_translator, walker2d_policies
 
@@ -109,16 +114,22 @@ from .mujoco import ant_translator, ant_policies
 REGISTRY["invertedPendulum_init_translator"] = invertedPendulum_translator.GameDescriber
 REGISTRY["invertedPendulum_basic_translator"] = invertedPendulum_translator.BasicStateSequenceTranslator
 REGISTRY["invertedPendulum_policies"] = [invertedPendulum_policies.pseudo_random_policy, invertedPendulum_policies.real_random_policy]
-
 REGISTRY["invertedDoublePendulum_init_translator"] = invertedDoublePendulum_translator.GameDescriber
 REGISTRY["invertedDoublePendulum_basic_translator"] = invertedDoublePendulum_translator.BasicStateSequenceTranslator
 REGISTRY["invertedDoublePendulum_policies"] = [invertedDoublePendulum_policies.pseudo_random_policy, invertedDoublePendulum_policies.real_random_policy]
 
 
+REGISTRY["swimmer_init_translator"] = swimmer_translator.GameDescriber
+REGISTRY["swimmer_basic_translator"] = swimmer_translator.BasicStateSequenceTranslator
+REGISTRY["swimmer_policies"] = [swimmer_policies.pseudo_random_policy, swimmer_policies.real_random_policy]
+
+REGISTRY["reacher_init_translator"] = reacher_translator.GameDescriber
+REGISTRY["reacher_basic_translator"] = reacher_translator.BasicStateSequenceTranslator
+REGISTRY["reacher_policies"] = [reacher_policies.pseudo_random_policy, reacher_policies.real_random_policy]
+
 REGISTRY["hopper_init_translator"] = hopper_translator.GameDescriber
 REGISTRY["hopper_basic_translator"] = hopper_translator.BasicStateSequenceTranslator
 REGISTRY["hopper_policies"] = [hopper_policies.pseudo_random_policy, hopper_policies.real_random_policy]
-
 REGISTRY["walker2d_init_translator"] = walker2d_translator.GameDescriber
 REGISTRY["walker2d_basic_translator"] = walker2d_translator.BasicStateSequenceTranslator
 REGISTRY["walker2d_policies"] = [walker2d_policies.pseudo_random_policy, walker2d_policies.real_random_policy]

envs/mujoco/reacher_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(2)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(2)]

envs/mujoco/reacher_translator.py

@@ -0,0 +1,67 @@
+'''Reacher
+Action Space Box(-1.0, 1.0, (2,), float32)
+
+Observation Space Box(-inf, inf, (11,), float64)
+'''
+class BasicLevelTranslator:
+    def __init__(self):
+        pass
+
+    def translate(self, state):
+        (cos_angle_arm1, cos_angle_arm2, sin_angle_arm1, sin_angle_arm2, 
+         target_x, target_y, angular_vel_arm1, angular_vel_arm2, 
+         diff_x, diff_y, diff_z) = state
+
+        res = (f"Arm1 has a cosine angle of {cos_angle_arm1:.2f} and a sine angle of {sin_angle_arm1:.2f}. "\
+               f"Arm2 has a cosine angle of {cos_angle_arm2:.2f} and a sine angle of {sin_angle_arm2:.2f}. "\
+               f"Target position is at ({target_x:.2f}, {target_y:.2f}). "\
+               f"Arm1's angular velocity is {angular_vel_arm1:.2f} rad/s, and Arm2's is {angular_vel_arm2:.2f} rad/s. "\
+               f"Vector difference between fingertip and target is ({diff_x:.2f}, {diff_y:.2f}, {diff_z:.2f}).")
+        return res
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+        }
+        self.reward_desc_dict = {
+        }
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len): 
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+
+    def describe_goal(self):
+        return "The goal is to control a two-jointed robot arm to move its end effector (fingertip) close to a randomly spawned target."
+
+    def describe_game(self):
+        return ("In the Reacher game, you control a two-jointed robot arm. The objective is to maneuver the arm's fingertip close to a target. "\
+                "The observation space includes the cosine and sine of the arm angles, coordinates of the target, angular velocities of the arms, "\
+                "and the vector from the fingertip to the target. The episode ends after 50 timesteps or if any state space value becomes non-finite. "\
+                "Rewards are given based on the distance of the fingertip from the target and the magnitude of actions applied.")
+
+    def describe_action(self):
+        return ("Your next move: \n Please provide two numerical values representing the torques applied at the two hinge joints. "\
+                "Each value should be within the range of [-1, 1].")
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = ("Take Action: Apply Torque at Joint 1: {:.2f}, "
+                           "Joint 2 Torque: {:.2f}"
+                          ).format(info['action'][0], info['action'][1])
+
+            reward_desc = f"Result: Reward of {info['reward']:.2f}, "
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(f"{state_desc}.\\n {action_desc} \\n {reward_desc} \\n Transit to {next_state_desc}")
+        return descriptions

envs/mujoco/swimmer_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(2)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(2)]

envs/mujoco/swimmer_translator.py

@@ -0,0 +1,80 @@
+'''Swimmer
+Action Space Box(-1.0, 1.0, (2,), float32)
+
+Observation Space Box(-inf, inf, (8,), float64)
+'''
+
+class BasicLevelTranslator:
+    def translate(self, state):
+        res = (
+            f"Angle of the front tip: {state[0]:.2f} rad\n"
+            f"Angle of the first rotor: {state[1]:.2f} rad\n"
+            f"Angle of the second rotor: {state[2]:.2f} rad\n"
+            f"Velocity of the tip along the x-axis: {state[3]:.2f} m/s\n"
+            f"Velocity of the tip along the y-axis: {state[4]:.2f} m/s\n"
+            f"Angular velocity of front tip: {state[5]:.2f} rad/s\n"
+            f"Angular velocity of the first rotor: {state[6]:.2f} rad/s\n"
+            f"Angular velocity of the second rotor: {state[7]:.2f} rad/s"
+        )
+        return res
+
+class GameDescriber:
+
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+        }
+        self.reward_desc_dict = {
+        }
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len): 
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+
+    def describe_goal(self):
+        return (
+            "The goal in the Swimmer environment is to move as fast as possible towards the right "\
+            "by applying torque to the rotors and utilizing fluid friction. The swimmer consists of "\
+            "three or more segments connected by rotors, and the objective is to achieve efficient "\
+            "swimming motion."
+        )
+
+    def describe_game(self):
+        return (
+            "In the Swimmer environment, you control a swimmer consisting of three or more segments "\
+            "connected by rotors. Your goal is to make the swimmer move as fast as possible to the right "\
+            "in a two-dimensional pool. You can achieve this by applying torques to the rotors and utilizing "\
+            "fluid friction. The environment provides observations of the swimmer's angles, velocities, "\
+            "and angular velocities."
+        )
+
+    def describe_action(self):
+        return (
+            "Your next move: \nPlease provide a list of two numerical values, each within the range of [-1, 1], "\
+            "representing the torques to be applied to the two rotors of the swimmer. These torques will help "\
+            "control the swimmer's movement and achieve efficient swimming."
+        )
+
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = (
+                "Torques Applied: "
+                f"Rotor 1: {info['action'][0]:.2f}, Rotor 2: {info['action'][1]:.2f}"
+            )
+            reward_desc = f"Result: Reward of {info['reward']:.2f}"
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(
+                f"{state_desc}\n{action_desc}\n{reward_desc}\nTransit to\n{next_state_desc}"
+            )
+        return descriptions

record_reflexion.csv

@@ -10,5 +10,10 @@ FrozenLake-v1,1,expert,200.0
 MountainCarContinuous-v0,1,expert,200.0
 RepresentedBoxing-v0,1,expert,200.0
 RepresentedPong-v0,1,expert,200.0
-Ant-v4,1,expert,5000
+Ant-v4,1,expert,5000.2
+HalfCheetah-v4,1,expert,12138.8
+Hopper-v4,1,expert,3542.2
+Walker2d-v4,1,expert,5000.0
+Swimmer-v4,1,expert,44.4
+Reacher-v4,1,expert,-2.6
 

shell/test_mujoco_ant.sh

@@ -1,6 +1,18 @@
 
 # Ant-v4
 
+# COT
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+
 # REFLEXION
 python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
 

shell/test_mujoco_invertedDoublePendulum.sh

@@ -1,5 +1,16 @@
 # InvertedDoublePendulum-v4
 
+# COT
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
 # REFLEXION
 python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
 

shell/test_mujoco_invertedPendulum.sh

@@ -1,16 +1,25 @@
 # InvertedPendulum-v4
 
+# COT
+python main_reflexion.py --env_name InvertedPendulum-v1 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name InvertedPendulum-v1 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name InvertedPendulum-v1 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name InvertedPendulum-v1 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+
 # REFLEXION
 python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
 
 python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
 
-python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
 
 
 # exe
 python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
 
 python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
-
-python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_reacher.sh

@@ -0,0 +1,27 @@
+# Reacher-v4
+
+# COT
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+# REFLEXION
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_swimmer.sh

@@ -0,0 +1,27 @@
+# Swimmer-v4
+
+# COT
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+# REFLEXION
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_walker2d.sh

@@ -1,5 +1,17 @@
 # Walker2d-v4
 
+# COT
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+
 # REFLEXION
 python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller