update gradio

app.py

@@ -94,8 +94,15 @@ class DemoRunner:
         self.simulation_runner._md_logger = ''
         # progress(0.2)
         yield "Task Generating ==>", None, None
+
+        t1 = time.time()
         yield from self.simulation_runner.example_task_creation()
         yield from self.simulation_runner.simulate_task()
+        # self.simulation_runner.example_task_creation()
+        # self.simulation_runner.simulate_task()
+
+        t2 = time.time()
+        print("run example cost = ", t2 - t1, " s")
 
 
 def setup(api_key, option_choice, target_task_name):
@@ -159,7 +166,7 @@ if __name__ == '__main__':
                 generated_task = gr.Markdown(label='Generated Task')
                 generated_asset = gr.Markdown(label='Generated Asset')
                 generated_code = gr.Code(label='Generated Code',  language="python", interactive=True)
-                video_run = gr.Video(label='Video of Last Instruction')
+                video_run = gr.Video(label='Video of Last Instruction', autoplay=True)
         btn_setup.click(
             setup,
             inputs=[inp_api_key, option_choice, inp_instruction],

cliport/cfg/data.yaml

@@ -30,5 +30,5 @@ record:
   add_text: False
   add_task_text: True
   fps: 20
-  video_height: 640
-  video_width: 720
+  video_height: 320
+  video_width: 360

cliport/environments/environment.py

@@ -307,6 +307,7 @@ class Environment(gym.Env):
                     color, depth, _ = self.render_camera(config)
                     obs['color'] += (color,)
                     obs['depth'] += (depth,)
+
                 return obs, 0.0, True, self.info
 
         start_time = time.time()
@@ -325,30 +326,33 @@ class Environment(gym.Env):
 
         obs = self._get_obs()
 
-        if not os.path.exists(self.record_cfg['save_video_path']):
-            os.mkdir(self.record_cfg['save_video_path'])
-        self.video_path = os.path.join(self.record_cfg['save_video_path'], "123.mp4")
-        video_writer = imageio.get_writer(self.video_path,
-                                          fps=self.record_cfg['fps'],
-                                          format='FFMPEG',
-                                          codec='h264', )
-        print(f"has {len(self.curr_video)} frames to save")
-        for color in self.curr_video:
-           video_writer.append_data(color)
-        print("save video to ", self.video_path)
-        video_writer.close()
+        # if not os.path.exists(self.record_cfg['save_video_path']):
+        #     os.mkdir(self.record_cfg['save_video_path'])
+        # self.video_path = os.path.join(self.record_cfg['save_video_path'], "123.mp4")
+        # video_writer = imageio.get_writer(self.video_path,
+        #                                   fps=self.record_cfg['fps'],
+        #                                   format='FFMPEG',
+        #                                   codec='h264', )
+        # print(f"has {len(self.curr_video)} frames to save")
+        # for color in self.curr_video:
+        #    video_writer.append_data(color)
+        # video_writer.close()
+        # print("save video to ", self.video_path)
+        # self.video_path = None
+
+
         self.cur_obs = obs
         self.cur_reward = reward
         self.cur_done = done
         self.cur_info = info
-        yield "Task Generated ==> Asset Generated ==> API Reviewed ==> Error Reviewed ==> Code Generated ==> Running Simulation", self.generated_code, self.video_path
+        # yield "Task Generated ==> Asset Generated ==> API Reviewed ==> Error Reviewed ==> Code Generated ==> Running Simulation", self.generated_code, self.video_path
 
 
     def step_simulation(self):
         p.stepSimulation()
         self.step_counter += 1
 
-        if self.save_video and self.step_counter % 5 == 0:
+        if self.save_video and self.step_counter % 60 == 0:
             self.add_video_frame()
 
     def render(self, mode='rgb_array'):
@@ -360,6 +364,62 @@ class Environment(gym.Env):
 
         return color
 
+    def render_camera_without_seg(self, config, image_size=None, shadow=1):
+        """Render RGB-D image with specified camera configuration."""
+        if not image_size:
+            image_size = config['image_size']
+
+        # OpenGL camera settings.
+        lookdir = np.float32([0, 0, 1]).reshape(3, 1)
+        updir = np.float32([0, -1, 0]).reshape(3, 1)
+        rotation = p.getMatrixFromQuaternion(config['rotation'])
+        rotm = np.float32(rotation).reshape(3, 3)
+        lookdir = (rotm @ lookdir).reshape(-1)
+        updir = (rotm @ updir).reshape(-1)
+        lookat = config['position'] + lookdir
+        focal_len = config['intrinsics'][0]
+        znear, zfar = config['zrange']
+        viewm = p.computeViewMatrix(config['position'], lookat, updir)
+        fovh = (image_size[0] / 2) / focal_len
+        fovh = (640 / 2) / focal_len
+        fovh = 180 * np.arctan(fovh) * 2 / np.pi
+
+        # Notes: 1) FOV is vertical FOV 2) aspect must be float
+        aspect_ratio = image_size[1] / image_size[0]
+        projm = p.computeProjectionMatrixFOV(fovh, aspect_ratio, znear, zfar)
+
+        # Render with OpenGL camera settings.
+        _, _, color, depth, segm = p.getCameraImage(
+            width=image_size[1],
+            height=image_size[0],
+            viewMatrix=viewm,
+            projectionMatrix=projm,
+            shadow=shadow,
+            flags=p.ER_SEGMENTATION_MASK_OBJECT_AND_LINKINDEX,
+            renderer=p.ER_BULLET_HARDWARE_OPENGL)
+
+        # Get color image.
+        color_image_size = (image_size[0], image_size[1], 4)
+        color = np.array(color, dtype=np.uint8).reshape(color_image_size)
+        color = color[:, :, :3]  # remove alpha channel
+        if config['noise']:
+            color = np.int32(color)
+            color += np.int32(self._random.normal(0, 3, image_size))
+            color = np.uint8(np.clip(color, 0, 255))
+
+        # Get depth image.
+        depth_image_size = (image_size[0], image_size[1])
+        zbuffer = np.array(depth).reshape(depth_image_size)
+        depth = (zfar + znear - (2. * zbuffer - 1.) * (zfar - znear))
+        depth = (2. * znear * zfar) / depth
+        if config['noise']:
+            depth += self._random.normal(0, 0.003, depth_image_size)
+
+        # Get segmentation image.
+        segm = np.uint8(segm).reshape(depth_image_size)
+
+        return color, depth, segm
+
     def render_camera(self, config, image_size=None, shadow=1):
         """Render RGB-D image with specified camera configuration."""
         if not image_size:
@@ -513,67 +573,67 @@ class Environment(gym.Env):
         # Render frame.
         config = self.agent_cams[0]
         image_size = (self.record_cfg['video_height'], self.record_cfg['video_width'])
-        color, depth, _ = self.render_camera(config, image_size, shadow=0)
+        color, _, _ = self.render_camera_without_seg(config, image_size, shadow=0)
         color = np.array(color)
 
-        if hasattr(self.record_cfg, 'blender_render') and  self.record_cfg['blender_render']:
-            # print("add blender key frame")
-            self.blender_recorder.add_keyframe()
+        # if hasattr(self.record_cfg, 'blender_render') and  self.record_cfg['blender_render']:
+        #     print("add blender key frame")
+            # self.blender_recorder.add_keyframe()
 
         # Add language instruction to video.
-        if self.record_cfg['add_text']:
-            lang_goal = self.get_lang_goal()
-            reward = f"Success: {self.task.get_reward():.3f}"
-
-            font = cv2.FONT_HERSHEY_DUPLEX
-            font_scale = 0.65
-            font_thickness =  1
-
-            # Write language goal.
-            line_length = 60
-            for i in range(len(lang_goal) // line_length + 1):
-                lang_textsize = cv2.getTextSize(lang_goal[i*line_length:(i+1)*line_length], font, font_scale, font_thickness)[0]
-                lang_textX = (image_size[1] - lang_textsize[0]) // 2
-                color = cv2.putText(color, lang_goal[i*line_length:(i+1)*line_length], org=(lang_textX, 570+i*30), # 600
-                                fontScale=font_scale,
-                                fontFace=font,
-                                color=(0, 0, 0),
-                                thickness=font_thickness, lineType=cv2.LINE_AA)
-
-            ## Write Reward.
-            # reward_textsize = cv2.getTextSize(reward, font, font_scale, font_thickness)[0]
-            # reward_textX = (image_size[1] - reward_textsize[0]) // 2
-            #
-            # color = cv2.putText(color, reward, org=(reward_textX, 634),
-            #                     fontScale=font_scale,
-            #                     fontFace=font,
-            #                     color=(0, 0, 0),
-            #                     thickness=font_thickness, lineType=cv2.LINE_AA)
-
-            color = np.array(color)
-
-        if 'add_task_text' in self.record_cfg and self.record_cfg['add_task_text']:
-            lang_goal = self.get_task_name()
-            reward = f"Success: {self.task.get_reward():.3f}"
-
-            font = cv2.FONT_HERSHEY_DUPLEX
-            font_scale = 1
-            font_thickness =  2
-
-            # Write language goal.
-            lang_textsize = cv2.getTextSize(lang_goal, font, font_scale, font_thickness)[0]
-            lang_textX = (image_size[1] - lang_textsize[0]) // 2
-
-            color = cv2.putText(color, lang_goal, org=(lang_textX, 600),
-                                fontScale=font_scale,
-                                fontFace=font,
-                                color=(255, 0, 0),
-                                thickness=font_thickness, lineType=cv2.LINE_AA)
-
-            color = np.array(color)
+        # if self.record_cfg['add_text']:
+        #     lang_goal = self.get_lang_goal()
+        #     reward = f"Success: {self.task.get_reward():.3f}"
+        #
+        #     font = cv2.FONT_HERSHEY_DUPLEX
+        #     font_scale = 0.65
+        #     font_thickness =  1
+        #
+        #     # Write language goal.
+        #     line_length = 60
+        #     for i in range(len(lang_goal) // line_length + 1):
+        #         lang_textsize = cv2.getTextSize(lang_goal[i*line_length:(i+1)*line_length], font, font_scale, font_thickness)[0]
+        #         lang_textX = (image_size[1] - lang_textsize[0]) // 2
+        #         color = cv2.putText(color, lang_goal[i*line_length:(i+1)*line_length], org=(lang_textX, 570+i*30), # 600
+        #                         fontScale=font_scale,
+        #                         fontFace=font,
+        #                         color=(0, 0, 0),
+        #                         thickness=font_thickness, lineType=cv2.LINE_AA)
+        #
+        #     ## Write Reward.
+        #     # reward_textsize = cv2.getTextSize(reward, font, font_scale, font_thickness)[0]
+        #     # reward_textX = (image_size[1] - reward_textsize[0]) // 2
+        #     #
+        #     # color = cv2.putText(color, reward, org=(reward_textX, 634),
+        #     #                     fontScale=font_scale,
+        #     #                     fontFace=font,
+        #     #                     color=(0, 0, 0),
+        #     #                     thickness=font_thickness, lineType=cv2.LINE_AA)
+        #
+        #     color = np.array(color)
+        #
+        # if 'add_task_text' in self.record_cfg and self.record_cfg['add_task_text']:
+        #     lang_goal = self.get_task_name()
+        #     reward = f"Success: {self.task.get_reward():.3f}"
+        #
+        #     font = cv2.FONT_HERSHEY_DUPLEX
+        #     font_scale = 1
+        #     font_thickness =  2
+        #
+        #     # Write language goal.
+        #     lang_textsize = cv2.getTextSize(lang_goal, font, font_scale, font_thickness)[0]
+        #     lang_textX = (image_size[1] - lang_textsize[0]) // 2
+        #
+        #     color = cv2.putText(color, lang_goal, org=(lang_textX, 600),
+        #                         fontScale=font_scale,
+        #                         fontFace=font,
+        #                         color=(255, 0, 0),
+        #                         thickness=font_thickness, lineType=cv2.LINE_AA)
+        #
+        #     color = np.array(color)
 
         self.curr_video.append(color)
-        self.video_writer.append_data(color)
+        # self.video_writer.append_data(color)
 
     def movep(self, pose, speed=0.01):
         """Move UR5 to target end effector pose."""

gensim/sim_runner.py

@@ -5,6 +5,8 @@ from cliport import tasks
 from cliport.dataset import RavensDataset
 from cliport.environments.environment import Environment
 
+import imageio
+
 from pygments import highlight
 from pygments.lexers import PythonLexer
 from pygments.formatters import TerminalFormatter
@@ -164,80 +166,17 @@ class BuildWheel(Task):
 
             # self.generated_task = {'task-name': 'TASK_NAME_TEMPLATE', 'task-description': 'TASK_STRING_TEMPLATE', 'assets-used': ['ASSET_1', 'ASSET_2', Ellipsis]}
             print("generated_task\n", self.generated_task)
-
             yield "Task Generated ==>", None, None
-
             self.generated_asset = self.agent.propose_assets()
-
-            # self.generated_asset = {}
             print("generated_asset\n", self.generated_asset)
             yield "Task Generated ==> Asset Generated ==> ", None, None
-
             self.agent.api_review()
-
-
             yield "Task Generated ==> Asset Generated ==> API Reviewed ==> ", None, None
             self.critic.error_review(self.generated_task)
-
-
             yield "Task Generated ==> Asset Generated ==> API Reviewed ==> Error Reviewed ==> ", None, None
             self.generated_code, self.curr_task_name = self.agent.implement_task()
             self.task_asset_logs.append(self.generated_task["assets-used"])
             self.generated_task_name = self.generated_task["task-name"]
-
-            # self.curr_task_name = self.generated_task_name = 'BuildWheel'
-#
-#             self.generated_code = """
-# import numpy as np
-# from cliport.tasks.task import Task
-# from cliport.utils import utils
-#
-# class BuildWheel(Task):
-#
-#     def __init__(self):
-#         super().__init__()
-#         self.max_steps = 10
-#         self.lang_template = "Construct a wheel using blocks and a sphere. First, position eight blocks in a circular layout on the tabletop. Each block should be touching its two neighbors and colored in alternating red and blue. Then place a green sphere in the center of the circular layout, completing the wheel."
-#         self.task_completed_desc = "done building wheel."
-#         self.additional_reset()
-#
-#     def reset(self, env):
-#         super().reset(env)
-#
-#         # Add blocks.
-#         block_size = (0.04, 0.04, 0.04)
-#         block_urdf = 'block/block.urdf'
-#         block_colors = [utils.COLORS['red'], utils.COLORS['blue']]
-#         blocks = []
-#         for i in range(8):
-#             block_pose = self.get_random_pose(env, block_size)
-#             block_id = env.add_object(block_urdf, block_pose, color=block_colors[i % 2])
-#             blocks.append(block_id)
-#
-#         # Add sphere.
-#         sphere_size = (0.04, 0.04, 0.04)
-#         sphere_urdf = 'sphere/sphere.urdf'
-#         sphere_color = utils.COLORS['green']
-#         sphere_pose = ((0.5, 0.0, 0.0), (0,0,0,1)) # fixed pose
-#         sphere_id = env.add_object(sphere_urdf, sphere_pose, color=sphere_color)
-#
-#         # Goal: blocks are arranged in a circle and sphere is in the center.
-#         circle_radius = 0.1
-#         circle_center = (0, 0, block_size[2] / 2)
-#         angles = np.linspace(0, 2 * np.pi, 8, endpoint=False)
-#         block_poses = [(circle_center[0] + circle_radius * np.cos(angle),
-#                         circle_center[1] + circle_radius * np.sin(angle),
-#                         circle_center[2]) for angle in angles]
-#         block_poses = [(utils.apply(sphere_pose, pos), sphere_pose[1]) for pos in block_poses]
-#         self.add_goal(objs=blocks, matches=np.ones((8, 8)), targ_poses=block_poses, replace=False,
-#                 rotations=True, metric='pose', params=None, step_max_reward=8 / 9)
-#
-#         # Goal: sphere is in the center of the blocks.
-#         self.add_goal(objs=[sphere_id], matches=np.ones((1, 1)), targ_poses=[sphere_pose], replace=False,
-#                 rotations=False, metric='pose', params=None, step_max_reward=1 / 9)
-#
-#         self.lang_goals.append(self.lang_template)
-# """
             print("generated_code\n", self.generated_code)
             print("curr_task_name\n", self.curr_task_name)
             yield "Task Generated ==> Asset Generated ==> API Reviewed ==> Error Reviewed ==> Code Generated ==> ", self.generated_code, None
@@ -368,8 +307,8 @@ class BuildWheel(Task):
                 act = expert.act(obs, info)
                 episode.append((obs, act, reward, info))
                 lang_goal = info['lang_goal']
-                env.generated_code = self.generated_code
-                yield from env.step(act)
+
+                env.step(act)
 
                 obs, reward, done, info = env.cur_obs, env.cur_reward, env.cur_done, env.cur_info
                 total_reward += reward
@@ -380,7 +319,21 @@ class BuildWheel(Task):
 
             end_time = time.time()
             print("end sim, time used = ", end_time - start_time)
-            yield "Task Generated ==> Asset Generated ==> API Reviewed ==> Error Reviewed ==> Code Generated ==> Simulation Running completed", self.generated_code, env.video_path
+
+            if not os.path.exists(env.record_cfg['save_video_path']):
+                os.mkdir(env.record_cfg['save_video_path'])
+            self.video_path = os.path.join(env.record_cfg['save_video_path'], "123.mp4")
+            video_writer = imageio.get_writer(self.video_path,
+                                              fps=env.record_cfg['fps'],
+                                              format='FFMPEG',
+                                              codec='h264', )
+            print(f"has {len(env.curr_video)} frames to save")
+            for color in env.curr_video:
+                video_writer.append_data(color)
+            video_writer.close()
+            print("save video to ", self.video_path)
+
+            yield "Task Generated ==> Asset Generated ==> API Reviewed ==> Error Reviewed ==> Code Generated ==> Simulation Running completed", self.generated_code, self.video_path
             episode.append((obs, None, reward, info))
 
 

temp/BuildWheel_error.txt

@@ -1,4 +1,4 @@
 Traceback (most recent call last):
-  File "/home/baochen/Desktop/projects/GPT-CLIPort/gensim/sim_runner.py", line 293, in simulate_task
-    yield "Task Generated ==> Asset Generated ==> Code Generated ==> Running Simulation", self.generated_code, self.video_path
-AttributeError: 'SimulationRunner' object has no attribute 'video_path'
+  File "/home/baochen/Desktop/projects/GPT-CLIPort/gensim/sim_runner.py", line 339, in simulate_task
+    yield "Task Generated ==> Asset Generated ==> API Reviewed ==> Error Reviewed ==> Code Generated ==> Simulation Running completed", self.generated_code, env.video_path
+AttributeError: 'Environment' object has no attribute 'video_path'