diff --git a/.gitattributes b/.gitattributes index 0311eae82d7757d4763c74efc1d7d645c57a3895..34e2bb35794d14dd5f851c967b42f95255b7464d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -69,3 +69,42 @@ n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/4_eval_4_f16 n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 filter=lfs diff=lfs merge=lfs -text n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 filter=lfs diff=lfs merge=lfs -text n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/episode_rosbags/episode_3_2025-01-03-22-42-59.bag filter=lfs diff=lfs merge=lfs -text +n627ep2q/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/episode_rosbags/episode_4_2025-01-03-22-43-44.bag filter=lfs diff=lfs merge=lfs -text +n627ep2q/episode_rosbags/episode_1_2025-01-03-22-41-05.bag filter=lfs diff=lfs merge=lfs -text +52k33ku2/episode_rosbags/episode_4_2025-01-03-22-50-51.bag filter=lfs diff=lfs merge=lfs -text +52k33ku2/episode_rosbags/episode_3_2025-01-03-22-50-07.bag filter=lfs diff=lfs merge=lfs -text +z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 filter=lfs diff=lfs merge=lfs -text +z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 filter=lfs diff=lfs merge=lfs -text +z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/episode_rosbags/episode_0_2025-01-03-22-11-09.bag filter=lfs diff=lfs merge=lfs -text +z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 filter=lfs diff=lfs merge=lfs -text +z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/episode_rosbags/episode_2_2025-01-03-22-49-24.bag filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/episode_rosbags/episode_1_2025-01-03-22-12-09.bag filter=lfs diff=lfs merge=lfs -text +z9oszy8k/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +z9oszy8k/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +z9oszy8k/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +z9oszy8k/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +z9oszy8k/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/episode_rosbags/episode_3_2025-01-03-22-13-51.bag filter=lfs diff=lfs merge=lfs -text +52k33ku2/episode_rosbags/episode_1_2025-01-03-22-48-36.bag filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/episode_rosbags/episode_4_2025-01-03-22-14-35.bag filter=lfs diff=lfs merge=lfs -text +tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/episode_rosbags/episode_2_2025-01-03-22-12-52.bag filter=lfs diff=lfs merge=lfs -text +tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/episode_rosbags/episode_2_2025-01-03-20-11-57.bag filter=lfs diff=lfs merge=lfs -text +3cjc38bu/episode_rosbags/episode_2_2025-01-04-20-55-09.bag filter=lfs diff=lfs merge=lfs -text diff --git a/3cjc38bu/episode_rosbags/episode_2_2025-01-04-20-55-09.bag b/3cjc38bu/episode_rosbags/episode_2_2025-01-04-20-55-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..c63c9c61e5323e46f712a76623f832e99fea61ec --- /dev/null +++ b/3cjc38bu/episode_rosbags/episode_2_2025-01-04-20-55-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5952e795b62bc01ae6109398ea4c1678084e7f270d6607a3ddfb0591ff56d5c2 +size 1411225020 diff --git a/52k33ku2/episode_rosbags/episode_1_2025-01-03-22-48-36.bag b/52k33ku2/episode_rosbags/episode_1_2025-01-03-22-48-36.bag new file mode 100644 index 0000000000000000000000000000000000000000..16b40b345f5d9689e5ed189e701d97151fceb9e7 --- /dev/null +++ b/52k33ku2/episode_rosbags/episode_1_2025-01-03-22-48-36.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a79a9563a780bd9ec999e0baaaa1ca40ea36dfc991d443820bb6b95ce0db190e +size 1411906468 diff --git a/52k33ku2/episode_rosbags/episode_2_2025-01-03-22-49-24.bag b/52k33ku2/episode_rosbags/episode_2_2025-01-03-22-49-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..5cedfa1daf8706aef3c9c709907ba662d4afd8ba --- /dev/null +++ b/52k33ku2/episode_rosbags/episode_2_2025-01-03-22-49-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b60ec8f61d981a68ee48f68c19ec415d4267c635b6c8536207cda0ad37aa81da +size 1410179919 diff --git a/52k33ku2/episode_rosbags/episode_3_2025-01-03-22-50-07.bag b/52k33ku2/episode_rosbags/episode_3_2025-01-03-22-50-07.bag new file mode 100644 index 0000000000000000000000000000000000000000..f0d0f536a6c64c66fa76c90691ad6bc09af4007c --- /dev/null +++ b/52k33ku2/episode_rosbags/episode_3_2025-01-03-22-50-07.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5564a0882294b9ddb0002f0dbc1e77d9596d9930bdf861d1cb0d825f5149fb95 +size 1411997129 diff --git a/52k33ku2/episode_rosbags/episode_4_2025-01-03-22-50-51.bag b/52k33ku2/episode_rosbags/episode_4_2025-01-03-22-50-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..1f4cd82bd09b1a7fdb001ffe77e34e22a20a80ac --- /dev/null +++ b/52k33ku2/episode_rosbags/episode_4_2025-01-03-22-50-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f849960c18628fdd3e246a09e9fab430c1e8368101f6589106576107aad5788 +size 1409398169 diff --git a/5q1nh8a1/episode_rosbags/episode_0_2025-01-03-22-11-09.bag b/5q1nh8a1/episode_rosbags/episode_0_2025-01-03-22-11-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..aed134c4d75ef3ab1a1c3e8cd32899db7742f1f8 --- /dev/null +++ b/5q1nh8a1/episode_rosbags/episode_0_2025-01-03-22-11-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c62a216788da1e8558038770f4691e82912bad2a7e8b3c85da82dedcfcd5afc +size 1411073838 diff --git a/5q1nh8a1/episode_rosbags/episode_1_2025-01-03-22-12-09.bag b/5q1nh8a1/episode_rosbags/episode_1_2025-01-03-22-12-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..bad05a3a6bec5b6c6e8e43892cd0d148ce3df010 --- /dev/null +++ b/5q1nh8a1/episode_rosbags/episode_1_2025-01-03-22-12-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7b2e676069f78c700861a141218513137fdc7f4cdca7ffaa8169a80c9887fc8 +size 1415842434 diff --git a/5q1nh8a1/episode_rosbags/episode_2_2025-01-03-22-12-52.bag b/5q1nh8a1/episode_rosbags/episode_2_2025-01-03-22-12-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..f638c99632ed1f4ab74b4cd65ecd9a64ae4088a3 --- /dev/null +++ b/5q1nh8a1/episode_rosbags/episode_2_2025-01-03-22-12-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90d5c077dfc19f48f370888083424013277f6a7a861071f467d163de554724df +size 1410056447 diff --git a/5q1nh8a1/episode_rosbags/episode_3_2025-01-03-22-13-51.bag b/5q1nh8a1/episode_rosbags/episode_3_2025-01-03-22-13-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..a11fc63899b0ddfa39b639b6a4012c3a49cbf133 --- /dev/null +++ b/5q1nh8a1/episode_rosbags/episode_3_2025-01-03-22-13-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9f9b5f1e19138f12c7ed34da58e5ac1322b4f076cdeced14692a77be2554c8f +size 1407745422 diff --git a/5q1nh8a1/episode_rosbags/episode_4_2025-01-03-22-14-35.bag b/5q1nh8a1/episode_rosbags/episode_4_2025-01-03-22-14-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..bb861b5d508bb46531c0a9018e96a9c80c09bcd9 --- /dev/null +++ b/5q1nh8a1/episode_rosbags/episode_4_2025-01-03-22-14-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef1df88e6671653a37a3ff310a9c27f3442427b735f272b88c066dd13b710938 +size 1406842936 diff --git a/j4k0okun/.hydra/config.yaml b/j4k0okun/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/j4k0okun/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/j4k0okun/.hydra/hydra.yaml b/j4k0okun/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aea06440fffb6eef4dd9dabbd43a3395a48b29a3 --- /dev/null +++ b/j4k0okun/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/j4k0okun + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/j4k0okun/.hydra/overrides.yaml b/j4k0okun/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/j4k0okun/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/j4k0okun/eval_policy.log b/j4k0okun/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..56b51416687b61804a9e46d8396b9aa8df2a3678 --- /dev/null +++ b/j4k0okun/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-03 20:04:27,631][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:04:27,635][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-03 20:04:28,778][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/n627ep2q/.hydra/config.yaml b/n627ep2q/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/n627ep2q/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/n627ep2q/.hydra/overrides.yaml b/n627ep2q/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/n627ep2q/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/n627ep2q/episode_rosbags/episode_1_2025-01-03-22-41-05.bag b/n627ep2q/episode_rosbags/episode_1_2025-01-03-22-41-05.bag new file mode 100644 index 0000000000000000000000000000000000000000..ab16c8503698cdf66529d2658c7d62347d1ef35d --- /dev/null +++ b/n627ep2q/episode_rosbags/episode_1_2025-01-03-22-41-05.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae4b30a4abc12f667ace38fe67bb866720833ac0ea6318bc9edbbd96ffea6e24 +size 1409757428 diff --git a/n627ep2q/episode_rosbags/episode_3_2025-01-03-22-42-59.bag b/n627ep2q/episode_rosbags/episode_3_2025-01-03-22-42-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..40121535bcfc8905663ad4d1a32013b3219f5811 --- /dev/null +++ b/n627ep2q/episode_rosbags/episode_3_2025-01-03-22-42-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2860803df26eb9e8e796aaeaa2cb87a44695c97f386767a93767b9b6a1454db4 +size 1464630519 diff --git a/n627ep2q/episode_rosbags/episode_4_2025-01-03-22-43-44.bag b/n627ep2q/episode_rosbags/episode_4_2025-01-03-22-43-44.bag new file mode 100644 index 0000000000000000000000000000000000000000..fe5443b2374e774eadbc1b47fe354183ba2cf451 --- /dev/null +++ b/n627ep2q/episode_rosbags/episode_4_2025-01-03-22-43-44.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a99ae41610e524a27bb45a10f188eb2a51e5bbb6ea1131b7682118a7417f2c54 +size 1410625666 diff --git a/n627ep2q/eval_video/0_eval.mp4 b/n627ep2q/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d00577c27415ae0705d1c25c21a781465997940 --- /dev/null +++ b/n627ep2q/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e1ff6c41d792e337b8f2fa6e45e5487696aaa8ac1327d66fbdb6f2b3f268ba6 +size 1192169 diff --git a/n627ep2q/eval_video/1_eval.mp4 b/n627ep2q/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b03372be3c1942c26c1bd350f248821ecb67b018 --- /dev/null +++ b/n627ep2q/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcb69cc1c7293969286be93fceba12831456e52541fc39ea06bba60f47ef79d8 +size 1226892 diff --git a/n627ep2q/eval_video/2_eval.mp4 b/n627ep2q/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8911d97858d98545d94466889718f225367a67bd --- /dev/null +++ b/n627ep2q/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:528e4a4b9d24ffcf67ff9b2cad422ea9ebedc0fae9781bdfffc00dd2a80633a7 +size 1209710 diff --git a/n627ep2q/eval_video/3_eval.mp4 b/n627ep2q/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..327f73d3d964d0e5507ad1564869e75034298683 --- /dev/null +++ b/n627ep2q/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e26516847c67094fe8eb83034b418e767ba00743a12e83d37a71c57e9100cd4a +size 1325955 diff --git a/n627ep2q/eval_video/4_eval.mp4 b/n627ep2q/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e7699e5d4197bb54c2844aec76b508b9dc99ee6 --- /dev/null +++ b/n627ep2q/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f169542c93769083c88595cd6729c0d3b9ce8bf76f0863db22b2e8e952883514 +size 1294074 diff --git a/n627ep2q/tb/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 b/n627ep2q/tb/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 new file mode 100644 index 0000000000000000000000000000000000000000..d28adeb33baed78b7649e86316618b211be9cc37 --- /dev/null +++ b/n627ep2q/tb/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f2a7f547dee68358464cb722a71ff29bb279e37b96c72d80d423a2d2f5b4037 +size 6539 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0ed32e8a3a12fc7e3ec3e18a4d10971ee36ac1d --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c37410ba4ec4aefe0458e5e32b97cd88f9385e8b41d0ef2dbafe8e163dae1c15 +size 910102360 diff --git a/tmmu86ea/.hydra/config.yaml b/tmmu86ea/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/tmmu86ea/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/tmmu86ea/.hydra/hydra.yaml b/tmmu86ea/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4e0efe4db31b35b9f3ffca0d9d9c0f44c880e9d --- /dev/null +++ b/tmmu86ea/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/tmmu86ea/.hydra/overrides.yaml b/tmmu86ea/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/tmmu86ea/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/tmmu86ea/episode_rosbags/aligned_depth_to_color_K.npy b/tmmu86ea/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/tmmu86ea/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/tmmu86ea/episode_rosbags/cam_tf_world.npy b/tmmu86ea/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/tmmu86ea/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/tmmu86ea/episode_rosbags/color_K.npy b/tmmu86ea/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/tmmu86ea/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/tmmu86ea/episode_rosbags/depth_K.npy b/tmmu86ea/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/tmmu86ea/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/tmmu86ea/episode_rosbags/episode_2_2025-01-03-20-11-57.bag b/tmmu86ea/episode_rosbags/episode_2_2025-01-03-20-11-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..de546868578ccbdacd59fa1615a367111c9858df --- /dev/null +++ b/tmmu86ea/episode_rosbags/episode_2_2025-01-03-20-11-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fd8cd7863288e750187b8446e43ca25f0107f6147c1df62c8b792cdf420e3b8 +size 1413423892 diff --git a/tmmu86ea/eval_robot.log b/tmmu86ea/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..8a98253114fd941e71b532a83721ca1454a7825a --- /dev/null +++ b/tmmu86ea/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 20:09:31,012][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:09:31,014][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/tmmu86ea/eval_video/0_eval.mp4 b/tmmu86ea/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2a9799f7583d48717b76b2f4fd098df2e527cf7a --- /dev/null +++ b/tmmu86ea/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84a6dabcfe82d3667159844b43bdf9b9078989b9d8834c408ea9e567f8ede33e +size 1159864 diff --git a/tmmu86ea/eval_video/1_eval.mp4 b/tmmu86ea/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64e0b3ef54ce62194d662f61d9b7b272f900f148 --- /dev/null +++ b/tmmu86ea/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91c97d4f4d3e6b888a209d1dcb66aa101df1ef653d3274719665dbe67ae9313c +size 1156871 diff --git a/tmmu86ea/eval_video/2_eval.mp4 b/tmmu86ea/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..317bc599975fb331470a495090dc2dc06f8eb89c --- /dev/null +++ b/tmmu86ea/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68dcc088f8980ac14b8c5277d712bcef5cbb79a24c992a42f701c4fd12707ea2 +size 1163949 diff --git a/tmmu86ea/eval_video/3_eval.mp4 b/tmmu86ea/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0dfedac5aeb6a70a001e0362bc2ae6990a463205 --- /dev/null +++ b/tmmu86ea/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a78dabf1835350f9eca377726082556c84d3734dbc8d495ba4f82bc45aafb0f +size 1167675 diff --git a/tmmu86ea/eval_video/4_eval.mp4 b/tmmu86ea/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f830c19ed0f062838768ecdb744d08f05351b579 --- /dev/null +++ b/tmmu86ea/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a760bf7d8f336eab9d0b35e3c40e217822abde17077db4d773b22f6f789886d +size 1173619 diff --git a/tmmu86ea/tb/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 b/tmmu86ea/tb/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 new file mode 100644 index 0000000000000000000000000000000000000000..4e901b054cdbf523f28fbcfced9611705fa16603 --- /dev/null +++ b/tmmu86ea/tb/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37deb3722189d86c8e41062321fec2bb44923eba4d953ca91728b13128361ed0 +size 6539 diff --git a/tmmu86ea/wandb/debug-internal.log b/tmmu86ea/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7b56dfe19c520c69f0283194706febfcf8b77150 --- /dev/null +++ b/tmmu86ea/wandb/debug-internal.log @@ -0,0 +1,783 @@ +2025-01-03 20:09:36,994 INFO StreamThr :1963622 [internal.py:wandb_internal():85] W&B internal server running at pid: 1963622, started at: 2025-01-03 20:09:36.993796 +2025-01-03 20:09:36,995 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:09:37,008 INFO WriterThread:1963622 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/run-tmmu86ea.wandb +2025-01-03 20:09:37,008 DEBUG SenderThread:1963622 [sender.py:send():379] send: header +2025-01-03 20:09:37,016 DEBUG SenderThread:1963622 [sender.py:send():379] send: run +2025-01-03 20:09:37,250 INFO SenderThread:1963622 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files +2025-01-03 20:09:37,250 INFO SenderThread:1963622 [sender.py:_start_run_threads():1188] run started: tmmu86ea with start time 1735952976.995053 +2025-01-03 20:09:37,268 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:09:37,268 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:09:37,312 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:09:37,316 DEBUG HandlerThread:1963622 [system_info.py:__init__():26] System info init +2025-01-03 20:09:37,316 DEBUG HandlerThread:1963622 [system_info.py:__init__():41] System info init done +2025-01-03 20:09:37,316 INFO HandlerThread:1963622 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:09:37,317 INFO SystemMonitor:1963622 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:09:37,317 INFO HandlerThread:1963622 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:09:37,317 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:09:37,318 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:09:37,318 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:09:37,319 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:09:37,319 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:09:37,329 DEBUG HandlerThread:1963622 [system_info.py:probe():152] Probing system +2025-01-03 20:09:37,330 DEBUG HandlerThread:1963622 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:09:37,335 DEBUG HandlerThread:1963622 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:09:37,335 DEBUG HandlerThread:1963622 [system_info.py:probe():200] Probing system done +2025-01-03 20:09:37,335 DEBUG HandlerThread:1963622 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:09:37.329536', 'startedAt': '2025-01-04T01:09:36.985274', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.4535, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.953, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 725.9888801574707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:09:37,335 INFO HandlerThread:1963622 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:09:37,335 INFO HandlerThread:1963622 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:09:37,335 DEBUG HandlerThread:1963622 [system_info.py:_save_code():44] Saving code +2025-01-03 20:09:37,337 DEBUG HandlerThread:1963622 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:09:37,337 DEBUG HandlerThread:1963622 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:09:37,401 DEBUG HandlerThread:1963622 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:09:37,401 INFO HandlerThread:1963622 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:09:37,404 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:09:37,404 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:09:37,404 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:09:37,405 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:09:37,654 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:09:37,654 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:09:37,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:09:37,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:09:37,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:37,657 INFO HandlerThread:1963622 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/tb" + save: true +} +control { + relay_id: "14d0e77ccdda4224a53b65c76ace9adf" +} +_info { + stream_id: "tmmu86ea" +} + +2025-01-03 20:09:37,657 DEBUG HandlerThread:1963622 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:09:37,694 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/em5prqc5-diff.patch +2025-01-03 20:09:37,722 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/t9vqkeat-code/FISH/eval_robot.py +2025-01-03 20:09:37,904 DEBUG SenderThread:1963622 [sender.py:send():379] send: telemetry +2025-01-03 20:09:37,904 DEBUG SenderThread:1963622 [sender.py:send():379] send: tbrecord +2025-01-03 20:09:37,905 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:09:37,905 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 with policy live +2025-01-03 20:09:37,951 INFO wandb-upload_0:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/dkwn83ij-wandb-metadata.json +2025-01-03 20:09:38,112 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/lhqhujqg-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:09:38,252 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code/FISH/eval_robot.py +2025-01-03 20:09:38,252 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:09:38,252 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/diff.patch +2025-01-03 20:09:38,252 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/requirements.txt +2025-01-03 20:09:38,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:38,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-metadata.json +2025-01-03 20:09:38,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code +2025-01-03 20:09:38,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code/FISH +2025-01-03 20:09:38,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:39,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:40,251 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:40,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:41,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:42,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:42,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:43,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:44,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:45,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:46,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:47,072 DEBUG SenderThread:1963622 [sender.py:send():379] send: config +2025-01-03 20:09:47,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:48,080 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:48,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:48,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:49,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:50,254 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:50,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:51,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:52,255 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:52,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:09:52,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:09:52,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:53,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:53,752 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:54,255 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:54,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:55,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:56,256 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:56,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:57,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:58,257 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:58,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:59,461 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:59,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:00,257 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:00,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:01,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:02,258 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:02,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:03,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:04,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:04,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:05,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:06,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:07,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:10:07,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:10:07,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:08,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:09,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:09,780 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:10,260 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:10,260 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml +2025-01-03 20:10:10,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:11,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:12,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:13,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:14,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:15,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:15,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:16,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:17,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:18,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:19,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:20,661 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:21,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:21,661 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:22,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:10:22,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:10:22,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:23,661 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:24,661 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:25,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:26,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:26,752 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:27,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:28,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:29,074 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:10:29,074 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 with policy now +2025-01-03 20:10:29,074 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:10:29,075 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:10:29,075 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:10:29,076 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:10:29,265 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:10:29,265 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 +2025-01-03 20:10:29,265 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media +2025-01-03 20:10:29,266 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:10:29,266 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos +2025-01-03 20:10:29,337 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/ab9ne20r-media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 +2025-01-03 20:10:29,598 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/2olv4drx-media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 +2025-01-03 20:10:29,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:30,265 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:30,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:31,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:32,083 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:32,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:33,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:34,267 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:34,664 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:35,664 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:36,268 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:36,664 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:37,319 DEBUG SystemMonitor:1963622 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:10:37,320 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:10:37,320 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:37,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:10:37,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:10:37,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:38,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:39,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:40,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:41,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:42,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:42,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:43,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:44,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:45,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:46,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:47,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:48,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:48,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:49,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:50,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:51,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:52,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:10:52,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:10:52,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:53,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:53,719 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:54,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:55,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:56,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:57,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:58,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:59,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:59,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:00,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:01,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:02,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:03,276 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:11:03,564 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/wf7d7cb7-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:11:03,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:04,277 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:04,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:05,465 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:05,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:06,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:07,321 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:11:07,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:11:07,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:11:07,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:08,278 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:08,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:09,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:10,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:10,744 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:11,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:12,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:13,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:14,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:15,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:16,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:16,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:17,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:18,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:19,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:20,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:21,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:21,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:22,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:11:22,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:11:22,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:23,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:24,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:25,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:26,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:26,728 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:27,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:28,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:29,678 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:30,285 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:30,679 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:31,678 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:32,286 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:32,578 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:32,679 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:33,679 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:34,185 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:11:34,185 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 with policy now +2025-01-03 20:11:34,187 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:11:34,188 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:11:34,188 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:11:34,188 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:11:34,286 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:11:34,287 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:34,287 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 +2025-01-03 20:11:34,287 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:11:34,450 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/vov6gugu-media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 +2025-01-03 20:11:34,679 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:34,705 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/zxm8xmsm-media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 +2025-01-03 20:11:35,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:36,287 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:36,681 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:37,321 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:11:37,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:11:37,657 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:11:37,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:37,808 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:38,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:39,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:40,288 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:40,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:41,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:42,289 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:42,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:43,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:43,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:44,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:45,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:46,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:47,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:48,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:49,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:49,684 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:50,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:51,684 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:52,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:11:52,657 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:11:52,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:53,685 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:54,293 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:11:54,686 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:55,091 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:11:55,092 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:11:55,092 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:55,092 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:11:55,293 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:11:55,686 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:56,294 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:56,686 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:57,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:58,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:59,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:00,108 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:00,295 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:00,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:01,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:02,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:03,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:04,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:05,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:05,893 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:06,297 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:06,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:07,321 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:12:07,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:12:07,657 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:12:07,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:08,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:09,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:10,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:11,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:11,741 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:12,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:13,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:14,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:15,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:16,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:17,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:17,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:18,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:19,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:20,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:21,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:22,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:12:22,657 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:12:22,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:22,749 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:23,690 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:24,690 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:25,691 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:26,304 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:26,329 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:12:26,329 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 with policy now +2025-01-03 20:12:26,332 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:12:26,333 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:12:26,333 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:12:26,334 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:12:26,605 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/w5jvfz8z-media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 +2025-01-03 20:12:26,691 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:27,304 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:12:27,304 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 +2025-01-03 20:12:27,305 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:12:27,619 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/bekqy5l3-media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 +2025-01-03 20:12:27,696 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:28,305 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:28,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:28,693 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:29,694 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:30,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:31,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:32,306 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:32,698 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:33,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:33,885 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:34,307 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:34,699 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:35,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:36,700 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:37,322 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:12:37,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:12:37,658 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:12:37,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:38,700 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:39,167 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:39,308 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:12:39,554 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/e3hgz9mr-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:12:39,699 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:39,924 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:12:39,925 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:12:39,925 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:12:40,309 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:12:40,309 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:40,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:41,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:42,310 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:42,709 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:43,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:44,311 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:44,347 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:44,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:45,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:46,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:47,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:48,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:49,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:49,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:50,702 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:51,702 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:52,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:12:52,658 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:12:52,702 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:53,703 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:54,703 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:54,786 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:55,703 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:56,703 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:57,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:58,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:59,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:00,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:00,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:01,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:02,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:03,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:04,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:05,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:06,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:06,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:07,322 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:13:07,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:13:07,658 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:13:07,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:08,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:09,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:10,708 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:11,162 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:13:11,162 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 with policy now +2025-01-03 20:13:11,163 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:13:11,163 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:13:11,164 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:13:11,164 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:13:11,320 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:13:11,320 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 +2025-01-03 20:13:11,321 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:13:11,432 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/qy9abric-media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 +2025-01-03 20:13:11,691 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/oxql2ic8-media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 +2025-01-03 20:13:11,707 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:12,207 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:12,320 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:12,718 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:13,721 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:14,713 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:15,723 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:16,322 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:16,714 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:17,684 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:17,745 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:18,714 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:19,717 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:20,323 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:20,719 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:21,717 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:22,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:13:22,662 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:13:22,719 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:22,757 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:23,716 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:24,723 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:25,716 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:26,722 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:27,721 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:28,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:28,721 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:29,729 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:29,984 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:13:29,985 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:13:29,985 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:13:30,326 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:13:30,327 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:13:30,327 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:30,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:31,723 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:32,327 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:32,723 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:33,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:34,328 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:34,640 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:34,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:35,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:36,729 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:37,323 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:13:37,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:13:37,658 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:13:37,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:38,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:39,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:39,979 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:40,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:41,730 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:42,331 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:42,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:43,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:44,332 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:44,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:45,685 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:45,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:46,727 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:47,727 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:48,727 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:49,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:50,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:51,685 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:51,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:52,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:13:52,659 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:13:52,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:53,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:54,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:55,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:56,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:56,780 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:57,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:58,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:59,743 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:00,741 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:01,514 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:14:01,514 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 with policy now +2025-01-03 20:14:01,519 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:14:01,520 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:01,520 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:01,520 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:01,745 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:01,773 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/qi1gsn8j-media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 +2025-01-03 20:14:02,337 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:02,338 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:02,338 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 +2025-01-03 20:14:02,338 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:14:02,538 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:02,624 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/ta9af08f-media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 +2025-01-03 20:14:02,752 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:03,745 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:04,748 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:05,748 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:06,339 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:06,751 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:07,323 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:14:07,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:14:07,666 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:14:07,747 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:07,822 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:08,749 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:09,751 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:10,340 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:10,765 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:11,755 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:12,036 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:12,037 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:12,037 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:12,170 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:14:12,170 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:14:12,341 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:12,341 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:12,341 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 20:14:12,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media +2025-01-03 20:14:12,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table +2025-01-03 20:14:12,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval +2025-01-03 20:14:12,395 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:14:12,554 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/cygzagvo-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 20:14:12,585 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/kc2fwgkh-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:12,734 INFO wandb-upload_0:1963622 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0l23afp7 +2025-01-03 20:14:12,742 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:14:12,752 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:14:12,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:13,150 INFO SenderThread:1963622 [sender.py:send_request_log_artifact():1525] logged artifact run-tmmu86ea-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgyOTM2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2ODA3MQ==', 'latestArtifact': None}} +2025-01-03 20:14:13,150 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:14:13,150 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-03 20:14:13,151 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:13,151 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:13,151 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,152 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,152 DEBUG SenderThread:1963622 [sender.py:send():379] send: telemetry +2025-01-03 20:14:13,152 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,156 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,156 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,156 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,156 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,157 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,157 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,157 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,157 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,158 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,158 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,158 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,158 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,158 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,159 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,159 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,159 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,159 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,160 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,160 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,160 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,160 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,160 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:14:13,341 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:13,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:13,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json +2025-01-03 20:14:13,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval +2025-01-03 20:14:13,394 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/rcwyk4zs-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 20:14:13,535 INFO wandb-upload_0:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/tc56rwhq-media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json +2025-01-03 20:14:13,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:14,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:14,479 INFO wandb-upload_1:1963622 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp52e45lg1 +2025-01-03 20:14:14,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:14,865 INFO SenderThread:1963622 [sender.py:send_request_log_artifact():1525] logged artifact run-tmmu86ea-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgyOTM3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2ODA3NA==', 'latestArtifact': None}} +2025-01-03 20:14:14,865 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:14:14,865 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json with policy now +2025-01-03 20:14:14,865 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:14,865 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:14,866 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:15,100 INFO wandb-upload_0:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/57s86n7f-media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json +2025-01-03 20:14:15,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:15,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:16,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:16,998 DEBUG SenderThread:1963622 [sender.py:send():379] send: exit +2025-01-03 20:14:16,998 INFO SenderThread:1963622 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:14:16,998 INFO SenderThread:1963622 [sender.py:send_exit():588] handling runtime: 279 +2025-01-03 20:14:16,999 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:16,999 INFO SenderThread:1963622 [sender.py:send_exit():594] send defer +2025-01-03 20:14:16,999 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:16,999 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:14:16,999 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:16,999 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:14:16,999 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:14:17,000 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:17,000 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:14:17,000 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:17,000 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:14:17,000 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:14:17,000 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:17,000 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:14:17,000 INFO HandlerThread:1963622 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:14:17,001 DEBUG SystemMonitor:1963622 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:14:17,001 DEBUG SystemMonitor:1963622 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:14:17,001 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:14:17,001 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:14:17,052 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:14:17,052 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:14:17,053 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:14:17,053 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:17,053 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:14:17,053 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:14:17,053 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:14:17,053 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:17,053 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:14:17,054 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:17,054 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:14:17,054 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:14:17,054 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:17,054 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:14:17,343 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:18,343 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:24,043 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:24,043 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:24,043 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:14:24,044 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,044 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:14:24,044 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:14:24,044 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:24,044 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:24,044 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:24,044 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,045 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:14:24,045 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:24,045 DEBUG SenderThread:1963622 [sender.py:send():379] send: summary +2025-01-03 20:14:24,049 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:24,050 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,050 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:14:24,050 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:14:24,050 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,050 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:14:24,050 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,050 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:14:24,054 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:24,163 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:14:24,163 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,163 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:14:24,163 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,163 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:14:24,279 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:14:24,279 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,279 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:14:24,279 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,279 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:14:24,279 INFO SenderThread:1963622 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:14:24,282 INFO SenderThread:1963622 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:14:24,284 INFO SenderThread:1963622 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:14:24,293 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:14:24,293 DEBUG SenderThread:1963622 [sender.py:send():379] send: artifact +2025-01-03 20:14:24,293 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,296 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:14:24,346 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:24,346 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:24,346 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml +2025-01-03 20:14:24,504 INFO SenderThread:1963622 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzgyNDA5Mg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzgyNDA5Mg==', 'versionIndex': 406}}} +2025-01-03 20:14:24,504 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,504 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:14:24,504 INFO SenderThread:1963622 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:14:24,999 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml config.yaml +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json wandb-summary.json +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/requirements.txt requirements.txt +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:25,352 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log output.log +2025-01-03 20:14:25,352 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/diff.patch diff.patch +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:14:25,355 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:25,356 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:25,356 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:14:25,356 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:25,357 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:14:25,357 INFO SenderThread:1963622 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:14:25,539 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/requirements.txt +2025-01-03 20:14:25,546 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml +2025-01-03 20:14:25,549 INFO wandb-upload_0:1963622 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:25,609 INFO wandb-upload_4:1963622 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:25,669 INFO wandb-upload_3:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/ehqs7jzp-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:25,869 INFO Thread-11 (_thread_body):1963622 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:14:25,869 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:25,869 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:14:25,870 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:25,870 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:14:25,870 INFO SenderThread:1963622 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:14:25,873 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:14:25,873 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:25,873 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:14:25,873 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:25,873 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:14:25,874 INFO SenderThread:1963622 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:14:25,999 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:26,080 INFO SenderThread:1963622 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:14:26,080 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:14:26,080 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:26,080 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:26,080 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:14:26,081 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:26,081 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:14:26,081 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:14:26,081 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:26,081 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:14:26,081 DEBUG SenderThread:1963622 [sender.py:send():379] send: final +2025-01-03 20:14:26,081 DEBUG SenderThread:1963622 [sender.py:send():379] send: footer +2025-01-03 20:14:26,081 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:26,081 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:14:26,082 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:26,082 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:26,082 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:14:26,082 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:26,083 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:14:26,083 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:14:26,083 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:26,084 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:14:26,138 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:26,138 INFO MainThread:1963622 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:14:26,139 INFO MainThread:1963622 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:14:26,139 INFO MainThread:1963622 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:14:26,139 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:14:26,139 INFO HandlerThread:1963622 [handler.py:finish():882] shutting down handler +2025-01-03 20:14:27,083 INFO WriterThread:1963622 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/run-tmmu86ea.wandb +2025-01-03 20:14:27,138 INFO SenderThread:1963622 [sender.py:finish():1615] shutting down sender +2025-01-03 20:14:27,138 INFO SenderThread:1963622 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:14:27,138 INFO SenderThread:1963622 [file_pusher.py:join():175] waiting for file pusher diff --git a/tmmu86ea/wandb/debug.log b/tmmu86ea/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2751f037d39466c02bcf04d9ea38d4bdb4da8810 --- /dev/null +++ b/tmmu86ea/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Configure stats pid to 1963456 +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/settings +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug.log +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug-internal.log +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:09:36,988 INFO MainThread:1963456 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea'} +2025-01-03 20:09:36,988 INFO MainThread:1963456 [wandb_init.py:init():619] starting backend +2025-01-03 20:09:36,988 INFO MainThread:1963456 [wandb_init.py:init():623] setting up manager +2025-01-03 20:09:36,992 INFO MainThread:1963456 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:09:36,994 INFO MainThread:1963456 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:09:37,008 INFO MainThread:1963456 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:09:37,014 INFO MainThread:1963456 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:09:37,268 INFO MainThread:1963456 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:09:37,307 INFO MainThread:1963456 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:09:37,307 INFO MainThread:1963456 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:09:37,655 INFO MainThread:1963456 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:09:37,655 INFO MainThread:1963456 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:09:37,655 INFO MainThread:1963456 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:09:37,655 INFO MainThread:1963456 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:09:37,656 INFO MainThread:1963456 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:09:37,657 INFO MainThread:1963456 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/tb, True +2025-01-03 20:09:47,072 INFO MainThread:1963456 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:14:27,140 WARNING MsgRouterThr:1963456 [router.py:message_loop():77] message_loop has been closed diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code/FISH/eval_robot.py b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71e210358af99e85407b123b5dda68ac7a203880 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735952976 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/diff.patch b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..f7ff69267e735ffcb07f943041defb41c108b4be --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/diff.patch @@ -0,0 +1,54 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..bc385c8 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,16 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 ++ + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..0bf58e8 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1938_0', ++ '1939_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json new file mode 100644 index 0000000000000000000000000000000000000000..2f85f61c92088a8993fdf1bc1b36967832fa135d --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.25744009022795533, 0.9953789374731651, 0.284928768916229, 0.987118244073719, 0.4244327843340374, 0.9746108054165524, 0.6104677766340574, 0.9433402008421657]]} \ No newline at end of file diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..a8d24e7e19ddab81a190271c00226dbf213b3db7 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[9, 10, 0.9, 0.5200507640798522, 0.9982204752909094, 0.6232272982350897, 0.997309503046054, 0.6731266677033356, 0.9856723024469826, 0.6802807673451352, 0.9879804522685772]]} \ No newline at end of file diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2a9799f7583d48717b76b2f4fd098df2e527cf7a --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84a6dabcfe82d3667159844b43bdf9b9078989b9d8834c408ea9e567f8ede33e +size 1159864 diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64e0b3ef54ce62194d662f61d9b7b272f900f148 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91c97d4f4d3e6b888a209d1dcb66aa101df1ef653d3274719665dbe67ae9313c +size 1156871 diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..317bc599975fb331470a495090dc2dc06f8eb89c --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68dcc088f8980ac14b8c5277d712bcef5cbb79a24c992a42f701c4fd12707ea2 +size 1163949 diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0dfedac5aeb6a70a001e0362bc2ae6990a463205 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a78dabf1835350f9eca377726082556c84d3734dbc8d495ba4f82bc45aafb0f +size 1167675 diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f830c19ed0f062838768ecdb744d08f05351b579 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a760bf7d8f336eab9d0b35e3c40e217822abde17077db4d773b22f6f789886d +size 1173619 diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dffe9e0d804fe56cc8e8f2ed4d862d523dfa1cc2 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log @@ -0,0 +1,271 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735952987.072916]: resetting environment +[INFO] [1735952987.075892]: cleared current plan +[INFO] [1735952987.077652]: moving to home +[INFO] [1735952988.079756]: reached home +[INFO] [1735952988.081551]: reset action history +[INFO] [1735952990.790967]: environment reset +Starting episode 0 +[INFO] [1735952990.792690]: resetting environment +[INFO] [1735952990.796121]: cleared current plan +[INFO] [1735952990.797060]: moving to home +[INFO] [1735952991.799300]: reached home +[INFO] [1735952991.800476]: reset action history +[INFO] [1735952994.510044]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735952997.408240]: resetting environment +[INFO] [1735952997.415578]: cleared current plan +[INFO] [1735952997.416528]: moving to home +[INFO] [1735952998.419138]: reached home +[INFO] [1735952998.420652]: reset action history +[INFO] [1735953001.129456]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/episode_rosbags/episode_0_2025-01-03-20-10-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1963456.log +started roslaunch server http://158.130.50.37:40621/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1963726] +process[print_text-2]: started with pid [1963750] +process[rosbag_record-3]: started with pid [1963751] +started bagging! +For topic gripper_width: timestamp difference is -100259207 for nearest: 1735953007561647512 - target: 1735953007661906719 at idx 96 +[INFO] [1735953028.124862]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735953028.879008]: Stored episode 1. +[INFO] [1735953028.879542]: Saving video... +[INFO] [1735953029.063205]: Video saved! +Starting episode 1 +[INFO] [1735953029.076504]: resetting environment +[INFO] [1735953029.081339]: cleared current plan +[INFO] [1735953029.083048]: moving to home +[INFO] [1735953032.686894]: reached home +[INFO] [1735953032.687353]: reset action history +[INFO] [1735953035.396675]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735953062.419281]: resetting environment +[INFO] [1735953062.421669]: cleared current plan +[INFO] [1735953062.422022]: moving to home +[INFO] [1735953063.423732]: reached home +[INFO] [1735953063.424603]: reset action history +[INFO] [1735953066.133415]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/episode_rosbags/episode_1_2025-01-03-20-11-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1963456.log +started roslaunch server http://158.130.50.37:34249/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1963863] +process[print_text-5]: started with pid [1963864] +process[rosbag_record-6]: started with pid [1963888] +started bagging! +For topic gripper_width: timestamp difference is 104783699 for nearest: 1735953087761615440 - target: 1735953087656831741 at idx 249 +For topic gripper_width: timestamp difference is 106207648 for nearest: 1735953089528281251 - target: 1735953089422073603 at idx 249 +[INFO] [1735953093.115301]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1735953093.880638]: Stored episode 2. +[INFO] [1735953093.884753]: Saving video... +[INFO] [1735953094.163137]: Video saved! +Starting episode 2 +[INFO] [1735953094.187686]: resetting environment +[INFO] [1735953094.192358]: cleared current plan +[INFO] [1735953094.192581]: moving to home +[INFO] [1735953097.795408]: reached home +[INFO] [1735953097.798981]: reset action history +[INFO] [1735953100.529847]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735953114.111382]: resetting environment +[INFO] [1735953114.121805]: cleared current plan +[INFO] [1735953114.122106]: moving to home +[INFO] [1735953115.129252]: reached home +[INFO] [1735953115.129914]: reset action history +[INFO] [1735953117.837878]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/episode_rosbags/episode_2_2025-01-03-20-11-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1963456.log +started roslaunch server http://158.130.50.37:39935/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1963998] +process[print_text-8]: started with pid [1964022] +process[rosbag_record-9]: started with pid [1964023] +started bagging! +For topic gripper_width: timestamp difference is 100713229 for nearest: 1735953124028281903 - target: 1735953123927568674 at idx 84 +For topic gripper_width: timestamp difference is 101361619 for nearest: 1735953124828287707 - target: 1735953124726926088 at idx 96 +[INFO] [1735953144.821053]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735953145.634530]: Stored episode 3. +[INFO] [1735953145.635756]: Saving video... +[INFO] [1735953146.297130]: Video saved! +Starting episode 3 +[INFO] [1735953146.336838]: resetting environment +[INFO] [1735953146.352029]: cleared current plan +[INFO] [1735953146.352336]: moving to home +[WARN] [1735953146.160816]: Plan exhausted +[WARN] [1735953146.200680]: Plan exhausted +[WARN] [1735953146.240680]: Plan exhausted +[WARN] [1735953146.280826]: Plan exhausted +[WARN] [1735953146.321264]: Plan exhausted +[INFO] [1735953150.057834]: reached home +[INFO] [1735953150.062324]: reset action history +[INFO] [1735953152.841883]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735953159.193982]: resetting environment +[INFO] [1735953159.207188]: cleared current plan +[INFO] [1735953159.210020]: moving to home +[INFO] [1735953160.214798]: reached home +[INFO] [1735953160.215603]: reset action history +[INFO] [1735953162.939345]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/episode_rosbags/episode_3_2025-01-03-20-12-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1963456.log +started roslaunch server http://158.130.50.37:46777/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1964136] +process[print_text-11]: started with pid [1964137] +process[rosbag_record-12]: started with pid [1964161] +started bagging! +[INFO] [1735953190.062054]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735953190.891905]: Stored episode 4. +[INFO] [1735953190.894488]: Saving video... +[INFO] [1735953191.129607]: Video saved! +Starting episode 4 +[INFO] [1735953191.170552]: resetting environment +[INFO] [1735953191.203545]: cleared current plan +[INFO] [1735953191.205797]: moving to home +[INFO] [1735953194.913662]: reached home +[INFO] [1735953194.919734]: reset action history +[INFO] [1735953197.694792]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735953209.355199]: resetting environment +[INFO] [1735953209.385080]: cleared current plan +[INFO] [1735953209.386859]: moving to home +[INFO] [1735953210.390920]: reached home +[INFO] [1735953210.394912]: reset action history +[INFO] [1735953213.170653]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/episode_rosbags/episode_4_2025-01-03-20-13-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1963456.log +started roslaunch server http://158.130.50.37:39485/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1964299] +process[print_text-14]: started with pid [1964300] +process[rosbag_record-15]: started with pid [1964301] +started bagging! +For topic gripper_width: timestamp difference is 101145095 for nearest: 1735953219961613244 - target: 1735953219860468149 at idx 91 +For topic gripper_width: timestamp difference is -160251968 for nearest: 1735953222261643298 - target: 1735953222421895266 at idx 124 +[INFO] [1735953240.411285]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735953241.221865]: Stored episode 5. +[INFO] [1735953241.225513]: Saving video... +[INFO] [1735953241.471924]: Video saved! +[INFO] [1735953241.521481]: resetting environment +[INFO] [1735953241.533543]: cleared current plan +[INFO] [1735953241.537783]: moving to home +[INFO] [1735953245.250775]: reached home +[INFO] [1735953245.264537]: reset action history +[INFO] [1735953248.052105]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/requirements.txt b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-metadata.json b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d80790c8448403c1a5e32b14240c3fdaee3eae4c --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T01:09:37.329536", + "startedAt": "2025-01-04T01:09:36.985274", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3748.4535, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.953, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 725.9888801574707 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..217939f623e392a42d09a652d2a01ad72afbaa3e --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "84a6dabcfe82d3667159844b43bdf9b9078989b9d8834c408ea9e567f8ede33e", "size": 1159864, "path": "media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4"}, "global_step": 665, "_timestamp": 1735953252.6492906, "_runtime": 275.6542375087738, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "91c97d4f4d3e6b888a209d1dcb66aa101df1ef653d3274719665dbe67ae9313c", "size": 1156871, "path": "media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4"}, "eval/num_success": 4.0, "episode": 10.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.25744009017944336, "eval/success_rate_ub_0.01": 0.9953789114952087, "eval/success_rate_lb_0.025": 0.2849287688732147, "eval/success_rate_ub_0.025": 0.9871182441711426, "eval/success_rate_lb_0.05": 0.42443278431892395, "eval/success_rate_ub_0.05": 0.9746108055114746, "eval/success_rate_lb_0.1": 0.610467791557312, "eval/success_rate_ub_0.1": 0.9433401823043823, "eval/2_eval": {"_type": "video-file", "sha256": "68dcc088f8980ac14b8c5277d712bcef5cbb79a24c992a42f701c4fd12707ea2", "size": 1163949, "path": "media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "2a78dabf1835350f9eca377726082556c84d3734dbc8d495ba4f82bc45aafb0f", "size": 1167675, "path": "media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "7a760bf7d8f336eab9d0b35e3c40e217822abde17077db4d773b22f6f789886d", "size": 1173619, "path": "media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "f6f07faa4a586efd9baa9e785d4017c5140aeeafdfeb020c33b4bb37a7d266af", "size": 397, "artifact_path": "wandb-client-artifact://hidixcew35sp76h0fj2mludf1qr0xhe5bs30ejxolq95trhy77bm3f16quebf1474v2lmb8akf30wdvunu9c3fn1ev6cw08v0irg5vp2jcqmemqulqsv2sw4mbhznqdz/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vk1h5o1e4qt5ntsh65fwxaynwusbxjuwbj3fzv5ibjjyhs9qonaymns0fdraelyf0sizq4eofktf4ss0a3hwgcyaa5ks5ynnexbmq9b3kqwx15ykzuh8g0qy04wg71pl:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 9, "total_num_episodes": 10, "total_success_rate": 0.9, "total_success_rate_lb_0.01": 0.5200507640798522, "total_success_rate_ub_0.01": 0.9982204752909094, "total_success_rate_lb_0.025": 0.6232272982350897, "total_success_rate_ub_0.025": 0.997309503046054, "total_success_rate_lb_0.05": 0.6731266677033356, "total_success_rate_ub_0.05": 0.9856723024469826, "total_success_rate_lb_0.1": 0.6802807673451352, "total_success_rate_ub_0.1": 0.9879804522685772, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "f1a19274e10d66387b7f2a34d13bf6e4eb0c9a8927ceb375e95a34281c2856a4", "size": 503, "artifact_path": "wandb-client-artifact://p0r0ustuepseade4qsms3dtkmic780axocphfgg9qhisiw6cixdwx20iy9gcs1qqyqcvm5celmxmh87ys8wrtfrbxzenc77ib9skyt1my9lapr2jtoq98377kb2ht61s/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://z6az5rk3scjit3weww0fi29bxw7281qlxnlyr431lpckoear9wxcupftbjdfaiwj72y3wz621yof40zjtdykuezab3vmczhce43w66ocmknjksoa5l4bvf2xrb11siw7:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 279}, "eval/total_success_rate": 0.8999999761581421, "eval/total_success_rate_lb_0.01": 0.5200507640838623, "eval/total_success_rate_ub_0.01": 0.9982205033302307, "eval/total_success_rate_lb_0.025": 0.6232272982597351, "eval/total_success_rate_ub_0.025": 0.9973095059394836, "eval/total_success_rate_lb_0.05": 0.6731266379356384, "eval/total_success_rate_ub_0.05": 0.9856722950935364, "eval/total_success_rate_lb_0.1": 0.6802807450294495, "eval/total_success_rate_ub_0.1": 0.9879804253578186} \ No newline at end of file diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug-internal.log b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7b56dfe19c520c69f0283194706febfcf8b77150 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug-internal.log @@ -0,0 +1,783 @@ +2025-01-03 20:09:36,994 INFO StreamThr :1963622 [internal.py:wandb_internal():85] W&B internal server running at pid: 1963622, started at: 2025-01-03 20:09:36.993796 +2025-01-03 20:09:36,995 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:09:37,008 INFO WriterThread:1963622 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/run-tmmu86ea.wandb +2025-01-03 20:09:37,008 DEBUG SenderThread:1963622 [sender.py:send():379] send: header +2025-01-03 20:09:37,016 DEBUG SenderThread:1963622 [sender.py:send():379] send: run +2025-01-03 20:09:37,250 INFO SenderThread:1963622 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files +2025-01-03 20:09:37,250 INFO SenderThread:1963622 [sender.py:_start_run_threads():1188] run started: tmmu86ea with start time 1735952976.995053 +2025-01-03 20:09:37,268 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:09:37,268 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:09:37,312 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:09:37,316 DEBUG HandlerThread:1963622 [system_info.py:__init__():26] System info init +2025-01-03 20:09:37,316 DEBUG HandlerThread:1963622 [system_info.py:__init__():41] System info init done +2025-01-03 20:09:37,316 INFO HandlerThread:1963622 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:09:37,317 INFO SystemMonitor:1963622 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:09:37,317 INFO HandlerThread:1963622 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:09:37,317 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:09:37,318 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:09:37,318 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:09:37,319 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:09:37,319 INFO SystemMonitor:1963622 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:09:37,329 DEBUG HandlerThread:1963622 [system_info.py:probe():152] Probing system +2025-01-03 20:09:37,330 DEBUG HandlerThread:1963622 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:09:37,335 DEBUG HandlerThread:1963622 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:09:37,335 DEBUG HandlerThread:1963622 [system_info.py:probe():200] Probing system done +2025-01-03 20:09:37,335 DEBUG HandlerThread:1963622 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:09:37.329536', 'startedAt': '2025-01-04T01:09:36.985274', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.4535, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.953, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 725.9888801574707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:09:37,335 INFO HandlerThread:1963622 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:09:37,335 INFO HandlerThread:1963622 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:09:37,335 DEBUG HandlerThread:1963622 [system_info.py:_save_code():44] Saving code +2025-01-03 20:09:37,337 DEBUG HandlerThread:1963622 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:09:37,337 DEBUG HandlerThread:1963622 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:09:37,401 DEBUG HandlerThread:1963622 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:09:37,401 INFO HandlerThread:1963622 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:09:37,404 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:09:37,404 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:09:37,404 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:09:37,405 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:09:37,654 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:09:37,654 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:09:37,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:09:37,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:09:37,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:37,657 INFO HandlerThread:1963622 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/tb" + save: true +} +control { + relay_id: "14d0e77ccdda4224a53b65c76ace9adf" +} +_info { + stream_id: "tmmu86ea" +} + +2025-01-03 20:09:37,657 DEBUG HandlerThread:1963622 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:09:37,694 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/em5prqc5-diff.patch +2025-01-03 20:09:37,722 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/t9vqkeat-code/FISH/eval_robot.py +2025-01-03 20:09:37,904 DEBUG SenderThread:1963622 [sender.py:send():379] send: telemetry +2025-01-03 20:09:37,904 DEBUG SenderThread:1963622 [sender.py:send():379] send: tbrecord +2025-01-03 20:09:37,905 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:09:37,905 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 with policy live +2025-01-03 20:09:37,951 INFO wandb-upload_0:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/dkwn83ij-wandb-metadata.json +2025-01-03 20:09:38,112 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/lhqhujqg-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:09:38,252 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code/FISH/eval_robot.py +2025-01-03 20:09:38,252 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:09:38,252 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/diff.patch +2025-01-03 20:09:38,252 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/requirements.txt +2025-01-03 20:09:38,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:38,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-metadata.json +2025-01-03 20:09:38,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code +2025-01-03 20:09:38,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code/FISH +2025-01-03 20:09:38,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:39,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:40,251 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:40,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:41,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:42,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:42,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:43,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:44,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:45,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:46,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:47,072 DEBUG SenderThread:1963622 [sender.py:send():379] send: config +2025-01-03 20:09:47,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:48,080 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:48,253 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:48,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:49,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:50,254 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:50,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:51,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:52,255 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:52,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:09:52,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:09:52,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:53,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:53,752 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:54,255 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:54,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:55,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:56,256 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:56,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:57,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:58,257 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:09:58,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:59,461 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:59,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:00,257 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:00,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:01,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:02,258 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:02,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:03,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:04,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:04,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:05,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:06,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:07,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:10:07,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:10:07,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:08,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:09,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:09,780 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:10,260 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:10,260 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml +2025-01-03 20:10:10,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:11,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:12,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:13,659 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:14,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:15,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:15,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:16,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:17,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:18,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:19,660 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:20,661 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:21,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:21,661 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:22,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:10:22,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:10:22,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:23,661 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:24,661 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:25,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:26,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:26,752 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:27,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:28,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:29,074 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:10:29,074 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 with policy now +2025-01-03 20:10:29,074 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:10:29,075 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:10:29,075 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:10:29,076 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:10:29,265 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:10:29,265 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 +2025-01-03 20:10:29,265 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media +2025-01-03 20:10:29,266 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:10:29,266 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos +2025-01-03 20:10:29,337 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/ab9ne20r-media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 +2025-01-03 20:10:29,598 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/2olv4drx-media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 +2025-01-03 20:10:29,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:30,265 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:30,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:31,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:32,083 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:32,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:33,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:34,267 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:34,664 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:35,664 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:36,268 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:10:36,664 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:37,319 DEBUG SystemMonitor:1963622 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:10:37,320 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:10:37,320 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:37,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:10:37,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:10:37,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:38,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:39,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:40,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:41,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:42,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:42,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:43,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:44,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:45,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:46,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:47,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:48,657 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:48,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:49,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:50,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:51,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:52,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:10:52,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:10:52,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:53,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:53,719 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:54,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:55,667 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:56,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:57,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:58,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:10:59,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:10:59,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:00,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:01,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:02,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:03,276 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:11:03,564 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/wf7d7cb7-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:11:03,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:04,277 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:04,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:05,465 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:05,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:06,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:07,321 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:11:07,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:11:07,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:11:07,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:08,278 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:08,669 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:09,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:10,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:10,744 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:11,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:12,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:13,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:14,670 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:15,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:16,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:16,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:17,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:18,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:19,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:20,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:21,663 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:21,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:22,655 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:11:22,655 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:11:22,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:23,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:24,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:25,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:26,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:26,728 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:27,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:28,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:29,678 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:30,285 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:30,679 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:31,678 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:32,286 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:32,578 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:32,679 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:33,679 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:34,185 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:11:34,185 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 with policy now +2025-01-03 20:11:34,187 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:11:34,188 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:11:34,188 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:11:34,188 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:11:34,286 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:11:34,287 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:34,287 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 +2025-01-03 20:11:34,287 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:11:34,450 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/vov6gugu-media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 +2025-01-03 20:11:34,679 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:34,705 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/zxm8xmsm-media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 +2025-01-03 20:11:35,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:36,287 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:36,681 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:37,321 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:11:37,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:11:37,657 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:11:37,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:37,808 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:38,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:39,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:40,288 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:40,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:41,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:42,289 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:42,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:43,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:43,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:44,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:45,682 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:46,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:47,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:48,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:49,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:49,684 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:50,683 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:51,684 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:52,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:11:52,657 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:11:52,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:53,685 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:54,293 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:11:54,686 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:55,091 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:11:55,092 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:11:55,092 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:11:55,092 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:11:55,293 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:11:55,686 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:56,294 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:11:56,686 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:57,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:58,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:11:59,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:00,108 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:00,295 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:00,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:01,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:02,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:03,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:04,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:05,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:05,893 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:06,297 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:06,687 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:07,321 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:12:07,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:12:07,657 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:12:07,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:08,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:09,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:10,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:11,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:11,741 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:12,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:13,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:14,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:15,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:16,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:17,666 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:17,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:18,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:19,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:20,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:21,689 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:22,656 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:12:22,657 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:12:22,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:22,749 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:23,690 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:24,690 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:25,691 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:26,304 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:26,329 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:12:26,329 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 with policy now +2025-01-03 20:12:26,332 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:12:26,333 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:12:26,333 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:12:26,334 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:12:26,605 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/w5jvfz8z-media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 +2025-01-03 20:12:26,691 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:27,304 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:12:27,304 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 +2025-01-03 20:12:27,305 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:12:27,619 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/bekqy5l3-media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 +2025-01-03 20:12:27,696 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:28,305 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:28,668 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:28,693 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:29,694 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:30,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:31,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:32,306 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:32,698 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:33,697 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:33,885 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:34,307 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:34,699 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:35,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:36,700 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:37,322 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:12:37,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:12:37,658 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:12:37,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:38,700 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:39,167 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:39,308 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:12:39,554 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/e3hgz9mr-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:12:39,699 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:39,924 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:12:39,925 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:12:39,925 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:12:40,309 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:12:40,309 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:40,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:41,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:42,310 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:42,709 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:43,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:44,311 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:12:44,347 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:44,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:45,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:46,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:47,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:48,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:49,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:49,701 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:50,702 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:51,702 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:52,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:12:52,658 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:12:52,702 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:53,703 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:54,703 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:54,786 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:12:55,703 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:56,703 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:57,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:58,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:12:59,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:00,671 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:00,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:01,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:02,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:03,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:04,704 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:05,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:06,672 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:06,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:07,322 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:13:07,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:13:07,658 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:13:07,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:08,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:09,705 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:10,708 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:11,162 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:13:11,162 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 with policy now +2025-01-03 20:13:11,163 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:13:11,163 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:13:11,164 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:13:11,164 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:13:11,320 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:13:11,320 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 +2025-01-03 20:13:11,321 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:13:11,432 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/qy9abric-media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 +2025-01-03 20:13:11,691 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/oxql2ic8-media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 +2025-01-03 20:13:11,707 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:12,207 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:12,320 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:12,718 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:13,721 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:14,713 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:15,723 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:16,322 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:16,714 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:17,684 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:17,745 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:18,714 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:19,717 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:20,323 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:20,719 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:21,717 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:22,662 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:13:22,662 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:13:22,719 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:22,757 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:23,716 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:24,723 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:25,716 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:26,722 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:27,721 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:28,688 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:28,721 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:29,729 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:29,984 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:13:29,985 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:13:29,985 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:13:30,326 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:13:30,327 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:13:30,327 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:30,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:31,723 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:32,327 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:32,723 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:33,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:34,328 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:34,640 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:34,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:35,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:36,729 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:37,323 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:13:37,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:13:37,658 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:13:37,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:38,725 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:39,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:39,979 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:40,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:41,730 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:42,331 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:42,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:43,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:44,332 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:13:44,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:45,685 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:45,726 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:46,727 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:47,727 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:48,727 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:49,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:50,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:51,685 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:51,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:52,658 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:13:52,659 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:13:52,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:53,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:54,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:55,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:56,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:56,780 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:13:57,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:58,731 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:13:59,743 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:00,741 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:01,514 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:14:01,514 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 with policy now +2025-01-03 20:14:01,519 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:14:01,520 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:01,520 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:01,520 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:01,745 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:01,773 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/qi1gsn8j-media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 +2025-01-03 20:14:02,337 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:02,338 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:02,338 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 +2025-01-03 20:14:02,338 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval +2025-01-03 20:14:02,538 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:02,624 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/ta9af08f-media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 +2025-01-03 20:14:02,752 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:03,745 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:04,748 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:05,748 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:06,339 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:06,751 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:07,323 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:14:07,665 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:14:07,666 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:14:07,747 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:07,822 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:08,749 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:09,751 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:10,340 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:10,765 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:11,755 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:12,036 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:12,037 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:12,037 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:12,170 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:14:12,170 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:14:12,341 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:12,341 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:12,341 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 20:14:12,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media +2025-01-03 20:14:12,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table +2025-01-03 20:14:12,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval +2025-01-03 20:14:12,395 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:14:12,554 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/cygzagvo-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 20:14:12,585 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/kc2fwgkh-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:12,734 INFO wandb-upload_0:1963622 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0l23afp7 +2025-01-03 20:14:12,742 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:14:12,752 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:14:12,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:13,150 INFO SenderThread:1963622 [sender.py:send_request_log_artifact():1525] logged artifact run-tmmu86ea-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgyOTM2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2ODA3MQ==', 'latestArtifact': None}} +2025-01-03 20:14:13,150 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:14:13,150 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-03 20:14:13,151 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:13,151 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:13,151 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,152 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,152 DEBUG SenderThread:1963622 [sender.py:send():379] send: telemetry +2025-01-03 20:14:13,152 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,156 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,156 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,156 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,156 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,157 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,157 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,157 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,157 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,158 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,158 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,158 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,158 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,158 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,159 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,159 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,159 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,159 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,160 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,160 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,160 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:13,160 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:13,160 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:14:13,341 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:13,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:13,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json +2025-01-03 20:14:13,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval +2025-01-03 20:14:13,394 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/rcwyk4zs-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 20:14:13,535 INFO wandb-upload_0:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/tc56rwhq-media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json +2025-01-03 20:14:13,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:14,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:14,479 INFO wandb-upload_1:1963622 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp52e45lg1 +2025-01-03 20:14:14,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:14,865 INFO SenderThread:1963622 [sender.py:send_request_log_artifact():1525] logged artifact run-tmmu86ea-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgyOTM3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2ODA3NA==', 'latestArtifact': None}} +2025-01-03 20:14:14,865 DEBUG SenderThread:1963622 [sender.py:send():379] send: files +2025-01-03 20:14:14,865 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json with policy now +2025-01-03 20:14:14,865 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:14,865 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:14,866 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:15,100 INFO wandb-upload_0:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/57s86n7f-media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json +2025-01-03 20:14:15,342 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:15,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:16,753 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:16,998 DEBUG SenderThread:1963622 [sender.py:send():379] send: exit +2025-01-03 20:14:16,998 INFO SenderThread:1963622 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:14:16,998 INFO SenderThread:1963622 [sender.py:send_exit():588] handling runtime: 279 +2025-01-03 20:14:16,999 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:16,999 INFO SenderThread:1963622 [sender.py:send_exit():594] send defer +2025-01-03 20:14:16,999 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:16,999 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:14:16,999 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:16,999 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:14:16,999 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:14:17,000 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:17,000 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:14:17,000 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:17,000 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:14:17,000 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:14:17,000 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:17,000 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:14:17,000 INFO HandlerThread:1963622 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:14:17,001 DEBUG SystemMonitor:1963622 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:14:17,001 DEBUG SystemMonitor:1963622 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:14:17,001 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:14:17,001 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:14:17,052 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:14:17,052 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:14:17,053 INFO HandlerThread:1963622 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:14:17,053 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:17,053 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:14:17,053 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:14:17,053 DEBUG SenderThread:1963622 [sender.py:send():379] send: stats +2025-01-03 20:14:17,053 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:17,053 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:14:17,054 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:17,054 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:14:17,054 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:14:17,054 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:17,054 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:14:17,343 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:18,343 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:24,043 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:24,043 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:24,043 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:14:24,044 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,044 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:14:24,044 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:14:24,044 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:24,044 DEBUG SenderThread:1963622 [sender.py:send():379] send: history +2025-01-03 20:14:24,044 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:14:24,044 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,045 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:14:24,045 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:24,045 DEBUG SenderThread:1963622 [sender.py:send():379] send: summary +2025-01-03 20:14:24,049 INFO SenderThread:1963622 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:14:24,050 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,050 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:14:24,050 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:14:24,050 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,050 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:14:24,050 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,050 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:14:24,054 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:14:24,163 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:14:24,163 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,163 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:14:24,163 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,163 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:14:24,279 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:14:24,279 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,279 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:14:24,279 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,279 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:14:24,279 INFO SenderThread:1963622 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:14:24,282 INFO SenderThread:1963622 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:14:24,284 INFO SenderThread:1963622 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:14:24,293 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:14:24,293 DEBUG SenderThread:1963622 [sender.py:send():379] send: artifact +2025-01-03 20:14:24,293 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:24,296 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:14:24,346 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:24,346 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:24,346 INFO Thread-12 :1963622 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml +2025-01-03 20:14:24,504 INFO SenderThread:1963622 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzgyNDA5Mg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzgyNDA5Mg==', 'versionIndex': 406}}} +2025-01-03 20:14:24,504 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:24,504 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:14:24,504 INFO SenderThread:1963622 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:14:24,999 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml config.yaml +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json wandb-summary.json +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/requirements.txt requirements.txt +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:14:25,347 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:25,352 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log output.log +2025-01-03 20:14:25,352 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/diff.patch diff.patch +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 media/videos/eval/1_eval_1_91c97d4f4d3e6b888a20.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 media/videos/eval/0_eval_0_84a6dabcfe82d3667159.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 media/videos/eval/3_eval_3_2a78dabf1835350f9eca.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 media/videos/eval/4_eval_4_7a760bf7d8f336eab9d0.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 media/videos/eval/2_eval_2_68dcc088f8980ac14b8c.mp4 +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json media/table/eval/total_success_rate_ci_6_f1a19274e10d66387b7f.table.json +2025-01-03 20:14:25,353 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:14:25,355 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:25,356 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:25,356 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:14:25,356 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:25,357 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:14:25,357 INFO SenderThread:1963622 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:14:25,539 INFO wandb-upload_2:1963622 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/requirements.txt +2025-01-03 20:14:25,546 INFO wandb-upload_1:1963622 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/config.yaml +2025-01-03 20:14:25,549 INFO wandb-upload_0:1963622 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/wandb-summary.json +2025-01-03 20:14:25,609 INFO wandb-upload_4:1963622 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/files/output.log +2025-01-03 20:14:25,669 INFO wandb-upload_3:1963622 [upload_job.py:push():130] Uploaded file /tmp/tmppq88rwknwandb/ehqs7jzp-events.out.tfevents.1735952977.leonmkim-ROG-Strix-G15CS-G15CS.1963456.0 +2025-01-03 20:14:25,869 INFO Thread-11 (_thread_body):1963622 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:14:25,869 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:25,869 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:14:25,870 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:25,870 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:14:25,870 INFO SenderThread:1963622 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:14:25,873 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:14:25,873 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:25,873 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:14:25,873 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:25,873 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:14:25,874 INFO SenderThread:1963622 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:14:25,999 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:26,080 INFO SenderThread:1963622 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:14:26,080 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:14:26,080 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:26,080 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:26,080 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:14:26,081 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:26,081 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:14:26,081 INFO SenderThread:1963622 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:14:26,081 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:14:26,081 INFO HandlerThread:1963622 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:14:26,081 DEBUG SenderThread:1963622 [sender.py:send():379] send: final +2025-01-03 20:14:26,081 DEBUG SenderThread:1963622 [sender.py:send():379] send: footer +2025-01-03 20:14:26,081 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: defer +2025-01-03 20:14:26,081 INFO SenderThread:1963622 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:14:26,082 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:26,082 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:26,082 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:14:26,082 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:14:26,083 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:14:26,083 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:14:26,083 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:14:26,084 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:14:26,138 DEBUG SenderThread:1963622 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:14:26,138 INFO MainThread:1963622 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:14:26,139 INFO MainThread:1963622 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:14:26,139 INFO MainThread:1963622 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:14:26,139 DEBUG HandlerThread:1963622 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:14:26,139 INFO HandlerThread:1963622 [handler.py:finish():882] shutting down handler +2025-01-03 20:14:27,083 INFO WriterThread:1963622 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/run-tmmu86ea.wandb +2025-01-03 20:14:27,138 INFO SenderThread:1963622 [sender.py:finish():1615] shutting down sender +2025-01-03 20:14:27,138 INFO SenderThread:1963622 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:14:27,138 INFO SenderThread:1963622 [file_pusher.py:join():175] waiting for file pusher diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug.log b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2751f037d39466c02bcf04d9ea38d4bdb4da8810 --- /dev/null +++ b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Configure stats pid to 1963456 +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/settings +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug.log +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/logs/debug-internal.log +2025-01-03 20:09:36,987 INFO MainThread:1963456 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:09:36,988 INFO MainThread:1963456 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea'} +2025-01-03 20:09:36,988 INFO MainThread:1963456 [wandb_init.py:init():619] starting backend +2025-01-03 20:09:36,988 INFO MainThread:1963456 [wandb_init.py:init():623] setting up manager +2025-01-03 20:09:36,992 INFO MainThread:1963456 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:09:36,994 INFO MainThread:1963456 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:09:37,008 INFO MainThread:1963456 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:09:37,014 INFO MainThread:1963456 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:09:37,268 INFO MainThread:1963456 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:09:37,307 INFO MainThread:1963456 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:09:37,307 INFO MainThread:1963456 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:09:37,655 INFO MainThread:1963456 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:09:37,655 INFO MainThread:1963456 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:09:37,655 INFO MainThread:1963456 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:09:37,655 INFO MainThread:1963456 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:09:37,656 INFO MainThread:1963456 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:09:37,657 INFO MainThread:1963456 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/tmmu86ea/tb, True +2025-01-03 20:09:47,072 INFO MainThread:1963456 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:14:27,140 WARNING MsgRouterThr:1963456 [router.py:message_loop():77] message_loop has been closed diff --git a/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/run-tmmu86ea.wandb b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/run-tmmu86ea.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4b1b42af6e7a356ae45a74ab70ee3d44364c76e3 Binary files /dev/null and b/tmmu86ea/wandb/run-20250103_200936-tmmu86ea/run-tmmu86ea.wandb differ diff --git a/u0i99mq0/episode_rosbags/aligned_depth_to_color_K.npy b/u0i99mq0/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/u0i99mq0/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/u0i99mq0/episode_rosbags/cam_tf_world.npy b/u0i99mq0/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/u0i99mq0/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/u0i99mq0/episode_rosbags/depth_K.npy b/u0i99mq0/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/u0i99mq0/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/u0i99mq0/eval_robot.log b/u0i99mq0/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..f7a54a66396e041fdb6c73d1c60d1c8a465d9666 --- /dev/null +++ b/u0i99mq0/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 22:40:24,529][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:40:24,531][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/z9oszy8k/.hydra/config.yaml b/z9oszy8k/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/z9oszy8k/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/z9oszy8k/.hydra/hydra.yaml b/z9oszy8k/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbc7f95502679e2d81f14d0aa871c5bdb85a38df --- /dev/null +++ b/z9oszy8k/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/z9oszy8k/.hydra/overrides.yaml b/z9oszy8k/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/z9oszy8k/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/z9oszy8k/episode_rosbags/aligned_depth_to_color_K.npy b/z9oszy8k/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/z9oszy8k/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/z9oszy8k/episode_rosbags/cam_tf_world.npy b/z9oszy8k/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/z9oszy8k/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/z9oszy8k/episode_rosbags/color_K.npy b/z9oszy8k/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/z9oszy8k/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/z9oszy8k/episode_rosbags/depth_K.npy b/z9oszy8k/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/z9oszy8k/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/z9oszy8k/eval_robot.log b/z9oszy8k/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..936d709c80561cfc59f5c5202d4da5385619a47f --- /dev/null +++ b/z9oszy8k/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 20:04:43,633][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:04:43,636][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/z9oszy8k/eval_video/0_eval.mp4 b/z9oszy8k/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..04c556121c9c8e2da45850fdc8c333243e891099 --- /dev/null +++ b/z9oszy8k/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:192a2c0c13410dee2aa944dc7eef13ece20b077d0713d7119342fc4687d1e1e6 +size 1148570 diff --git a/z9oszy8k/eval_video/1_eval.mp4 b/z9oszy8k/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..71bdb035b2a53ebfc1b054be91621bc883196ded --- /dev/null +++ b/z9oszy8k/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a412d87eb0226343aaaea9d885b6c94a5abef7ee07492554328ad1dc2e991a4b +size 1155637 diff --git a/z9oszy8k/eval_video/2_eval.mp4 b/z9oszy8k/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37e53ed960e196da1e311b34626bc90402081fa2 --- /dev/null +++ b/z9oszy8k/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83a3bb9782b59fb8ba2ce99e1619c55b2fcc1e796734ef06fe55875ebb53e6ab +size 1172260 diff --git a/z9oszy8k/eval_video/3_eval.mp4 b/z9oszy8k/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6dfddeec09011811418dc1fd74fc7b1bbbfad1e0 --- /dev/null +++ b/z9oszy8k/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9576179b6df8182760d0eb1da5898c09f6de5738679d4076e62a1041b64866bc +size 1153070 diff --git a/z9oszy8k/eval_video/4_eval.mp4 b/z9oszy8k/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70ee477ba76be677db88866f52dfa6868c2388c2 --- /dev/null +++ b/z9oszy8k/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:622f2c07b65747a9a125eb3c04737fd439f5724a19a78bee5bdb2cf28ac6e37f +size 1159733 diff --git a/z9oszy8k/tb/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 b/z9oszy8k/tb/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 new file mode 100644 index 0000000000000000000000000000000000000000..7ae4d30fbcae322132e0cf2b23205ed7d415796f --- /dev/null +++ b/z9oszy8k/tb/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8c31384de92e4bbdfe1fa2262ac79ba4070d9ecef1d665aa9f7cba65344de59 +size 6539 diff --git a/z9oszy8k/wandb/debug-internal.log b/z9oszy8k/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d2460a1aeebaa99df9838f7d8882352df8e97194 --- /dev/null +++ b/z9oszy8k/wandb/debug-internal.log @@ -0,0 +1,760 @@ +2025-01-03 20:04:49,383 INFO StreamThr :1962513 [internal.py:wandb_internal():85] W&B internal server running at pid: 1962513, started at: 2025-01-03 20:04:49.383219 +2025-01-03 20:04:49,385 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:04:49,396 INFO WriterThread:1962513 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/run-z9oszy8k.wandb +2025-01-03 20:04:49,396 DEBUG SenderThread:1962513 [sender.py:send():379] send: header +2025-01-03 20:04:49,407 DEBUG SenderThread:1962513 [sender.py:send():379] send: run +2025-01-03 20:04:49,632 INFO SenderThread:1962513 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files +2025-01-03 20:04:49,632 INFO SenderThread:1962513 [sender.py:_start_run_threads():1188] run started: z9oszy8k with start time 1735952689.38404 +2025-01-03 20:04:49,646 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:04:49,646 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:04:49,718 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:04:49,723 DEBUG HandlerThread:1962513 [system_info.py:__init__():26] System info init +2025-01-03 20:04:49,723 DEBUG HandlerThread:1962513 [system_info.py:__init__():41] System info init done +2025-01-03 20:04:49,723 INFO HandlerThread:1962513 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:04:49,723 INFO SystemMonitor:1962513 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:04:49,724 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:04:49,724 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:04:49,724 INFO HandlerThread:1962513 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:04:49,724 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:04:49,725 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:04:49,725 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:04:49,736 DEBUG HandlerThread:1962513 [system_info.py:probe():152] Probing system +2025-01-03 20:04:49,738 DEBUG HandlerThread:1962513 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:04:49,742 DEBUG HandlerThread:1962513 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:04:49,742 DEBUG HandlerThread:1962513 [system_info.py:probe():200] Probing system done +2025-01-03 20:04:49,743 DEBUG HandlerThread:1962513 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:04:49.736564', 'startedAt': '2025-01-04T01:04:49.376098', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3565.950875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.976, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4504.695, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 719.3927230834961}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:04:49,743 INFO HandlerThread:1962513 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:04:49,743 INFO HandlerThread:1962513 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:04:49,743 DEBUG HandlerThread:1962513 [system_info.py:_save_code():44] Saving code +2025-01-03 20:04:49,744 DEBUG HandlerThread:1962513 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:04:49,744 DEBUG HandlerThread:1962513 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:04:49,807 DEBUG HandlerThread:1962513 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:04:49,807 INFO HandlerThread:1962513 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:04:49,810 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:04:49,810 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:04:49,811 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:04:49,811 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:04:50,049 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:04:50,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:04:50,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:04:50,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:04:50,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:50,053 INFO HandlerThread:1962513 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/tb" + save: true +} +control { + relay_id: "7561cf02a9ee4f3f8920fa7c02089ede" +} +_info { + stream_id: "z9oszy8k" +} + +2025-01-03 20:04:50,053 DEBUG HandlerThread:1962513 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:04:50,077 INFO wandb-upload_1:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/as3goyvs-code/FISH/eval_robot.py +2025-01-03 20:04:50,078 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/spev4z88-wandb-metadata.json +2025-01-03 20:04:50,182 DEBUG SenderThread:1962513 [sender.py:send():379] send: telemetry +2025-01-03 20:04:50,182 DEBUG SenderThread:1962513 [sender.py:send():379] send: tbrecord +2025-01-03 20:04:50,182 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:04:50,183 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 with policy live +2025-01-03 20:04:50,389 INFO wandb-upload_1:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/qrn2cqly-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:04:50,633 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:04:50,633 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-metadata.json +2025-01-03 20:04:50,633 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/requirements.txt +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/diff.patch +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code/FISH/eval_robot.py +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code/FISH +2025-01-03 20:04:51,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:51,849 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/j6yptl3d-diff.patch +2025-01-03 20:04:52,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:52,632 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:04:53,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:54,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:55,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:04:55,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:56,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:57,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:58,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:59,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:00,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:00,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:01,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:02,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:03,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:03,524 DEBUG SenderThread:1962513 [sender.py:send():379] send: config +2025-01-03 20:05:04,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:04,636 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:05,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:05:05,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:05:05,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:05,131 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:06,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:06,636 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:07,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:08,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:08,637 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:09,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:10,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:10,638 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:10,962 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:11,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:12,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:12,638 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:13,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:14,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:14,639 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:15,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:16,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:16,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:16,640 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:17,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:18,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:18,640 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:19,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:20,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:05:20,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:05:20,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:21,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:21,140 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:21,641 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml +2025-01-03 20:05:22,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:23,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:24,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:25,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:26,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:26,642 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:27,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:27,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:28,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:29,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:30,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:31,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:32,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:33,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:33,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:34,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:35,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:05:35,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:05:35,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:36,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:37,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:38,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:38,205 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:39,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:40,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:41,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:42,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:43,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:44,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:44,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:45,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:45,375 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:05:45,376 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 with policy now +2025-01-03 20:05:45,376 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:05:45,378 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:05:45,379 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:05:45,380 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:05:45,625 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/nqi2tip8-media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 +2025-01-03 20:05:45,648 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:05:45,648 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 +2025-01-03 20:05:45,648 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:05:45,649 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos +2025-01-03 20:05:45,649 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media +2025-01-03 20:05:46,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:46,583 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/c75xshva-media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 +2025-01-03 20:05:46,648 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:47,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:48,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:49,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:49,092 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:49,726 DEBUG SystemMonitor:1962513 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:05:49,726 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:05:50,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:05:50,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:05:50,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:50,649 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:51,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:52,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:53,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:54,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:54,651 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:54,842 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:55,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:56,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:57,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:58,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:59,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:00,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:00,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:01,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:02,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:03,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:03,654 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:06:03,896 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/utc83uet-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:06:04,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:04,654 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:05,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:06:05,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:06:05,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:05,137 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:06,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:06,655 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:07,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:08,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:08,655 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:09,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:10,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:11,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:11,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:12,064 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:13,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:14,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:15,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:16,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:16,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:17,064 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:18,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:19,064 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:19,727 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:06:20,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:06:20,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:06:20,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:21,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:21,152 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:22,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:23,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:24,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:25,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:26,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:27,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:27,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:28,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:29,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:30,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:31,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:32,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:32,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:33,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:34,069 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:34,598 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:06:34,602 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:06:34,602 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 with policy now +2025-01-03 20:06:34,603 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:06:34,603 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:06:34,603 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:06:34,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:06:34,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:34,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 +2025-01-03 20:06:34,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:06:34,855 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/reahra3f-media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 +2025-01-03 20:06:35,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:06:35,052 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:06:35,071 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:35,143 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/7jb00nzn-media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 +2025-01-03 20:06:36,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:36,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:37,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:37,130 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:38,073 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:39,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:40,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:40,665 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:41,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:42,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:42,169 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:42,666 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:43,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:44,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:44,462 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:06:44,462 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:06:44,462 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:06:44,667 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:06:44,667 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:06:45,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:46,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:46,668 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:47,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:47,657 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:48,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:48,668 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:49,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:49,727 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:06:50,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:06:50,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:06:50,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:50,669 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:51,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:52,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:53,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:53,130 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:54,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:55,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:56,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:57,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:58,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:59,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:59,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:00,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:01,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:02,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:03,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:04,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:05,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:07:05,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:07:05,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:05,154 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:06,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:07,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:08,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:09,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:10,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:11,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:11,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:12,078 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:13,078 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:14,078 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:15,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:15,931 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:07:15,931 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 with policy now +2025-01-03 20:07:15,934 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:07:15,935 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:07:15,935 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:07:15,936 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:07:16,085 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:16,178 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/lhp9vxpr-media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 +2025-01-03 20:07:16,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:07:16,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:16,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 +2025-01-03 20:07:16,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:07:16,954 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/n6014vuj-media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 +2025-01-03 20:07:16,974 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:17,086 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:18,086 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:18,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:19,084 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:19,728 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:07:20,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:07:20,054 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:07:20,087 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:20,679 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:21,086 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:22,084 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:22,202 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:23,083 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:24,085 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:24,680 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:25,083 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:26,102 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:27,085 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:28,070 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:28,086 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:29,088 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:30,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:31,094 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:32,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:33,101 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:33,102 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:33,301 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:07:33,301 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:07:33,302 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:07:33,683 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:07:33,683 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:07:33,921 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/s2ld6lpz-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:07:34,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:34,684 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:35,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:07:35,062 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:07:35,105 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:36,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:36,684 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:37,105 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:38,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:38,308 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:38,685 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:39,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:40,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:41,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:42,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:43,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:44,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:44,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:45,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:46,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:47,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:48,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:49,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:49,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:49,728 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:07:50,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:07:50,054 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:07:50,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:51,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:52,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:53,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:54,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:54,222 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:55,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:56,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:57,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:58,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:59,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:00,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:00,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:01,100 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:02,100 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:03,100 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:04,118 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:05,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:08:05,060 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:08:05,109 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:05,141 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:05,341 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:08:05,341 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 with policy now +2025-01-03 20:08:05,342 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:08:05,342 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:08:05,343 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:08:05,344 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:08:05,614 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/j9wbowe1-media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 +2025-01-03 20:08:05,694 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:08:05,694 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 +2025-01-03 20:08:05,694 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:08:05,966 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/3v4wya38-media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 +2025-01-03 20:08:06,111 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:06,694 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:07,104 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:08,112 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:09,117 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:10,115 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:10,696 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:11,067 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:11,110 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:12,107 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:12,696 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:13,108 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:14,116 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:14,697 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:15,112 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:16,069 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:16,119 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:17,120 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:18,120 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:19,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:19,729 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:08:20,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:08:20,061 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:08:20,121 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:21,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:21,380 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:08:21,381 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:08:21,381 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:21,382 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:08:21,699 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:08:21,699 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:08:22,126 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:22,700 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:23,128 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:24,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:24,700 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:25,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:26,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:26,701 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:27,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:27,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:28,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:29,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:30,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:30,703 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:31,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:32,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:33,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:33,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:34,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:35,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:08:35,057 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:08:35,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:36,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:37,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:38,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:38,200 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:39,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:40,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:41,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:42,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:43,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:44,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:44,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:45,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:46,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:47,138 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:48,141 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:49,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:49,140 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:49,729 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:08:50,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:08:50,057 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:08:50,140 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:51,140 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:52,145 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:52,710 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:53,147 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:53,216 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:08:53,216 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 with policy now +2025-01-03 20:08:53,219 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:08:53,220 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:08:53,220 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:08:53,220 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:08:53,710 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:08:53,711 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 +2025-01-03 20:08:53,711 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:08:54,154 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:54,251 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:54,711 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:55,159 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:55,711 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:56,151 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:57,149 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:57,958 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/zgw01m04-media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 +2025-01-03 20:08:58,150 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:58,199 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/24jjo04m-media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 +2025-01-03 20:08:58,712 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:59,166 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:59,730 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:00,158 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:00,713 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:01,159 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:02,151 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:03,181 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:03,435 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:09:03,435 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:03,436 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:03,602 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:09:03,602 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table +2025-01-03 20:09:03,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval +2025-01-03 20:09:03,788 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:09:03,954 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/p0vddcs5-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:09:04,033 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/901biqc9-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:04,056 INFO wandb-upload_1:1962513 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdh9cniaz +2025-01-03 20:09:04,150 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:04,298 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:09:04,308 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:09:04,511 INFO SenderThread:1962513 [sender.py:send_request_log_artifact():1525] logged artifact run-z9oszy8k-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgyMzkxOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2NzM5OQ==', 'latestArtifact': None}} +2025-01-03 20:09:04,511 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:09:04,511 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 20:09:04,512 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:09:04,512 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,512 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,512 DEBUG SenderThread:1962513 [sender.py:send():379] send: telemetry +2025-01-03 20:09:04,512 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,513 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,513 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,513 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,515 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,515 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,516 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,516 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,516 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,517 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,517 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,517 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,517 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,517 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,518 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,518 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,518 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,518 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,519 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,519 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,519 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,519 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,520 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:09:04,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:04,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:04,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:04,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:09:04,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval +2025-01-03 20:09:04,737 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/0vqfzc2f-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:09:04,859 INFO wandb-upload_1:1962513 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppere7wef +2025-01-03 20:09:04,885 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/k2u43mkq-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:09:05,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:09:05,150 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:05,279 INFO SenderThread:1962513 [sender.py:send_request_log_artifact():1525] logged artifact run-z9oszy8k-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgyMzkzMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2NzQwMg==', 'latestArtifact': None}} +2025-01-03 20:09:05,279 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:09:05,279 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json with policy now +2025-01-03 20:09:05,279 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:09:05,279 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:05,280 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:05,280 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:05,280 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:09:05,514 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/r5golrpm-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:09:05,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:06,073 DEBUG SenderThread:1962513 [sender.py:send():379] send: exit +2025-01-03 20:09:06,073 INFO SenderThread:1962513 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:send_exit():588] handling runtime: 256 +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:send_exit():594] send defer +2025-01-03 20:09:06,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,074 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:09:06,074 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:09:06,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,075 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:09:06,075 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:06,075 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:09:06,075 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:09:06,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,075 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:09:06,075 INFO HandlerThread:1962513 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:09:06,075 DEBUG SystemMonitor:1962513 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:09:06,075 DEBUG SystemMonitor:1962513 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:09:06,076 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:09:06,076 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:09:06,147 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:09:06,148 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:09:06,148 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:09:06,148 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:06,148 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:09:06,148 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:09:06,148 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:09:06,149 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,149 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:09:06,149 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:06,149 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:09:06,149 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:09:06,149 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,149 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:09:06,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:06,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:08,717 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:11,718 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml +2025-01-03 20:09:12,441 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:12,441 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:12,441 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:09:12,442 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:12,442 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:09:12,442 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:09:12,442 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:12,443 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:09:12,443 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:12,443 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:09:12,443 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:12,444 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:12,444 DEBUG SenderThread:1962513 [sender.py:send():379] send: summary +2025-01-03 20:09:12,444 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:12,444 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:12,444 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:09:12,444 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:09:12,444 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:12,444 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:09:12,445 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:12,445 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:09:12,445 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:09:12,445 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:12,445 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:12,445 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:09:12,445 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:12,445 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:09:12,719 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:13,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:13,175 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:09:13,175 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:13,176 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:13,176 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:09:13,176 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:13,176 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:09:13,176 INFO SenderThread:1962513 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:09:13,178 INFO SenderThread:1962513 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:09:13,180 INFO SenderThread:1962513 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:09:13,191 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:09:13,191 DEBUG SenderThread:1962513 [sender.py:send():379] send: artifact +2025-01-03 20:09:13,194 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:13,194 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:09:13,664 INFO wandb-upload_2:1962513 [upload_job.py:push():85] Skipped uploading /tmp/tmpid9omv5h/wandb-job.json +2025-01-03 20:09:13,664 INFO wandb-upload_1:1962513 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr5ulhnd_ +2025-01-03 20:09:13,719 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:13,790 INFO wandb-upload_0:1962513 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpg0eiejcp +2025-01-03 20:09:14,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:14,296 INFO SenderThread:1962513 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzgyNDA5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzgxMTc2Nw==', 'versionIndex': 405}}} +2025-01-03 20:09:14,296 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:14,296 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:09:14,296 INFO SenderThread:1962513 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:09:14,720 INFO SenderThread:1962513 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files +2025-01-03 20:09:14,720 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml config.yaml +2025-01-03 20:09:14,720 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json wandb-summary.json +2025-01-03 20:09:14,720 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/requirements.txt requirements.txt +2025-01-03 20:09:14,722 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:09:14,722 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:14,722 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log output.log +2025-01-03 20:09:14,723 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/diff.patch diff.patch +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 +2025-01-03 20:09:14,727 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 +2025-01-03 20:09:14,727 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:09:14,727 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:09:14,727 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:09:14,727 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:14,727 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:14,729 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:09:14,729 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:14,729 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:09:14,729 INFO SenderThread:1962513 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:09:14,896 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml +2025-01-03 20:09:14,935 INFO wandb-upload_1:1962513 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:14,935 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/requirements.txt +2025-01-03 20:09:14,952 INFO wandb-upload_4:1962513 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:15,019 INFO wandb-upload_3:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/w688wnoc-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:15,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:15,076 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:15,220 INFO Thread-11 (_thread_body):1962513 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:09:15,220 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:15,220 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:09:15,220 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:15,220 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:09:15,220 INFO SenderThread:1962513 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:09:15,221 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:09:15,222 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:15,222 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:09:15,222 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:15,222 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:09:15,222 INFO SenderThread:1962513 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:09:15,286 INFO SenderThread:1962513 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:09:15,286 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:09:15,286 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:15,286 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:09:15,286 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:15,286 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:09:15,287 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:09:15,287 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:15,287 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:09:15,287 DEBUG SenderThread:1962513 [sender.py:send():379] send: final +2025-01-03 20:09:15,287 DEBUG SenderThread:1962513 [sender.py:send():379] send: footer +2025-01-03 20:09:15,287 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:15,287 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:09:15,288 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:15,288 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:09:15,288 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:15,288 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:15,288 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:09:15,289 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:09:15,289 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:09:15,291 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:15,349 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:15,350 INFO MainThread:1962513 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:09:15,350 INFO MainThread:1962513 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:09:15,350 INFO MainThread:1962513 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:09:15,351 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:09:15,351 INFO HandlerThread:1962513 [handler.py:finish():882] shutting down handler +2025-01-03 20:09:16,288 INFO WriterThread:1962513 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/run-z9oszy8k.wandb +2025-01-03 20:09:16,350 INFO SenderThread:1962513 [sender.py:finish():1615] shutting down sender +2025-01-03 20:09:16,350 INFO SenderThread:1962513 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:09:16,350 INFO SenderThread:1962513 [file_pusher.py:join():175] waiting for file pusher diff --git a/z9oszy8k/wandb/debug.log b/z9oszy8k/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ce1ef2ffe5a2e60310c23898c851ec238624c70c --- /dev/null +++ b/z9oszy8k/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Configure stats pid to 1962348 +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/settings +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug.log +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug-internal.log +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k'} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:init():619] starting backend +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:init():623] setting up manager +2025-01-03 20:04:49,382 INFO MainThread:1962348 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:04:49,383 INFO MainThread:1962348 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:04:49,395 INFO MainThread:1962348 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:04:49,405 INFO MainThread:1962348 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:04:49,646 INFO MainThread:1962348 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:04:49,714 INFO MainThread:1962348 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:04:49,714 INFO MainThread:1962348 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:04:50,051 INFO MainThread:1962348 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:04:50,051 INFO MainThread:1962348 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:04:50,051 INFO MainThread:1962348 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:04:50,051 INFO MainThread:1962348 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:04:50,052 INFO MainThread:1962348 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:04:50,052 INFO MainThread:1962348 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/tb, True +2025-01-03 20:05:03,524 INFO MainThread:1962348 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:09:16,351 WARNING MsgRouterThr:1962348 [router.py:message_loop():77] message_loop has been closed diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code/FISH/eval_robot.py b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3e12a1ed151aec76613b8f5d347cb2e52587f9a --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735952689 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/diff.patch b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..f7ff69267e735ffcb07f943041defb41c108b4be --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/diff.patch @@ -0,0 +1,54 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..bc385c8 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,16 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 ++ + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..0bf58e8 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1938_0', ++ '1939_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json new file mode 100644 index 0000000000000000000000000000000000000000..ebd62798ae2246ea155c51d2c7f88d715ecd78c3 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 5, 1.0, 0.43520843983992363, 1, 0.60740816591022, 1, 0.6842965185273651, 1, 0.6614799052153926, 1]]} \ No newline at end of file diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..04c556121c9c8e2da45850fdc8c333243e891099 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:192a2c0c13410dee2aa944dc7eef13ece20b077d0713d7119342fc4687d1e1e6 +size 1148570 diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..71bdb035b2a53ebfc1b054be91621bc883196ded --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a412d87eb0226343aaaea9d885b6c94a5abef7ee07492554328ad1dc2e991a4b +size 1155637 diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37e53ed960e196da1e311b34626bc90402081fa2 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83a3bb9782b59fb8ba2ce99e1619c55b2fcc1e796734ef06fe55875ebb53e6ab +size 1172260 diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6dfddeec09011811418dc1fd74fc7b1bbbfad1e0 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9576179b6df8182760d0eb1da5898c09f6de5738679d4076e62a1041b64866bc +size 1153070 diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70ee477ba76be677db88866f52dfa6868c2388c2 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:622f2c07b65747a9a125eb3c04737fd439f5724a19a78bee5bdb2cf28ac6e37f +size 1159733 diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2dc8b506215043f9dcc7c3ceabec3dcf9976fd96 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log @@ -0,0 +1,265 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735952703.524714]: resetting environment +[INFO] [1735952703.530359]: cleared current plan +[INFO] [1735952703.531250]: moving to home +[INFO] [1735952704.533283]: reached home +[INFO] [1735952704.534927]: reset action history +[INFO] [1735952707.243812]: environment reset +Starting episode 0 +[INFO] [1735952707.245623]: resetting environment +[INFO] [1735952707.248480]: cleared current plan +[INFO] [1735952707.250917]: moving to home +[INFO] [1735952708.253264]: reached home +[INFO] [1735952708.254512]: reset action history +[INFO] [1735952710.962296]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735952713.572498]: resetting environment +[INFO] [1735952713.575520]: cleared current plan +[INFO] [1735952713.576530]: moving to home +[INFO] [1735952714.578933]: reached home +[INFO] [1735952714.580844]: reset action history +[INFO] [1735952717.287671]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/episode_rosbags/episode_0_2025-01-03-20-05-17.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1962348.log +started roslaunch server http://158.130.50.37:42141/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1962709] +process[print_text-2]: started with pid [1962710] +process[rosbag_record-3]: started with pid [1962734] +started bagging! +For topic gripper_width: timestamp difference is 111365180 for nearest: 1735952724828290324 - target: 1735952724716925144 at idx 84 +[INFO] [1735952744.346175]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735952745.170722]: Stored episode 1. +[INFO] [1735952745.171118]: Saving video... +[INFO] [1735952745.364363]: Video saved! +Starting episode 1 +[INFO] [1735952745.377360]: resetting environment +[INFO] [1735952745.387690]: cleared current plan +[INFO] [1735952745.389084]: moving to home +[INFO] [1735952749.091490]: reached home +[INFO] [1735952749.091986]: reset action history +[INFO] [1735952751.798233]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735952762.888564]: resetting environment +[INFO] [1735952762.890884]: cleared current plan +[INFO] [1735952762.891040]: moving to home +[INFO] [1735952763.892465]: reached home +[INFO] [1735952763.892885]: reset action history +[INFO] [1735952766.601908]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/episode_rosbags/episode_1_2025-01-03-20-06-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1962348.log +started roslaunch server http://158.130.50.37:46819/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1962845] +process[print_text-5]: started with pid [1962846] +process[rosbag_record-6]: started with pid [1962847] +started bagging! +[INFO] [1735952793.579718]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735952794.339496]: Stored episode 2. +[INFO] [1735952794.339706]: Saving video... +[INFO] [1735952794.560555]: Video saved! +Starting episode 2 +[INFO] [1735952794.599040]: resetting environment +[INFO] [1735952794.623966]: cleared current plan +[INFO] [1735952794.633388]: moving to home +[INFO] [1735952798.344597]: reached home +[INFO] [1735952798.347991]: reset action history +[INFO] [1735952801.123338]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735952803.939293]: resetting environment +[INFO] [1735952803.946716]: cleared current plan +[INFO] [1735952803.946914]: moving to home +[INFO] [1735952804.948722]: reached home +[INFO] [1735952804.949214]: reset action history +[INFO] [1735952807.656528]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/episode_rosbags/episode_2_2025-01-03-20-06-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1962348.log +started roslaunch server http://158.130.50.37:40819/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1963002] +process[print_text-8]: started with pid [1963003] +process[rosbag_record-9]: started with pid [1963004] +started bagging! +For topic gripper_width: timestamp difference is -101727354 for nearest: 1735952834061622036 - target: 1735952834163349390 at idx 249 +[INFO] [1735952834.825251]: Storing episode... +[print_text-8] killing on exit +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[INFO] [1735952835.621113]: Stored episode 3. +[INFO] [1735952835.624101]: Saving video... +[INFO] [1735952835.901826]: Video saved! +Starting episode 3 +[INFO] [1735952835.938018]: resetting environment +[INFO] [1735952835.964710]: cleared current plan +[INFO] [1735952835.970902]: moving to home +[INFO] [1735952839.588224]: reached home +[INFO] [1735952839.590664]: reset action history +[INFO] [1735952842.377001]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735952853.075303]: resetting environment +[INFO] [1735952853.089984]: cleared current plan +[INFO] [1735952853.091695]: moving to home +[INFO] [1735952854.098467]: reached home +[INFO] [1735952854.100293]: reset action history +[INFO] [1735952857.058465]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/episode_rosbags/episode_3_2025-01-03-20-07-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1962348.log +started roslaunch server http://158.130.50.37:45083/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1963116] +process[print_text-11]: started with pid [1963117] +process[rosbag_record-12]: started with pid [1963118] +started bagging! +[INFO] [1735952884.027604]: Storing episode... +[rosbag_record-12] killing on exit[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1735952884.822469]: Stored episode 4. +[INFO] [1735952884.833478]: Saving video... +[INFO] [1735952885.288237]: Video saved! +Starting episode 4 +[INFO] [1735952885.344935]: resetting environment +[INFO] [1735952885.361751]: cleared current plan +[INFO] [1735952885.362324]: moving to home +[INFO] [1735952889.072157]: reached home +[INFO] [1735952889.074173]: reset action history +[INFO] [1735952891.876350]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735952901.002713]: resetting environment +[INFO] [1735952901.017849]: cleared current plan +[INFO] [1735952901.020492]: moving to home +[INFO] [1735952902.027128]: reached home +[INFO] [1735952902.028393]: reset action history +[INFO] [1735952904.806854]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/episode_rosbags/episode_4_2025-01-03-20-08-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1962348.log +started roslaunch server http://158.130.50.37:38763/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1963250] +process[print_text-14]: started with pid [1963251] +process[rosbag_record-15]: started with pid [1963275] +started bagging! +For topic EE_pose: timestamp difference is 287606605 for nearest: 1735952908789161094 - target: 1735952908501554489 at idx 0 +[INFO] [1735952931.795799]: Storing episode... +[pub_text-13] killing on exit +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1735952932.679677]: Stored episode 5. +[INFO] [1735952932.684879]: Saving video... +[INFO] [1735952933.175463]: Video saved! +[INFO] [1735952933.220275]: resetting environment +[INFO] [1735952933.245197]: cleared current plan +[INFO] [1735952933.250055]: moving to home +[WARN] [1735952933.151754]: Plan exhausted +[WARN] [1735952933.189264]: Plan exhausted +[WARN] [1735952933.229083]: Plan exhausted +[INFO] [1735952936.869711]: reached home +[INFO] [1735952936.876487]: reset action history +[INFO] [1735952939.729321]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/requirements.txt b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-metadata.json b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7c94ce142d09c09a248ee06602b8fbaddfd3910e --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T01:04:49.736564", + "startedAt": "2025-01-04T01:04:49.376098", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3565.950875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.976, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4504.695, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 719.3927230834961 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..384f25f22b9fa0db204c99bac9088711936b461d --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "192a2c0c13410dee2aa944dc7eef13ece20b077d0713d7119342fc4687d1e1e6", "size": 1148570, "path": "media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4"}, "global_step": 665, "_timestamp": 1735952944.2180178, "_runtime": 254.83397769927979, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "a412d87eb0226343aaaea9d885b6c94a5abef7ee07492554328ad1dc2e991a4b", "size": 1155637, "path": "media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "83a3bb9782b59fb8ba2ce99e1619c55b2fcc1e796734ef06fe55875ebb53e6ab", "size": 1172260, "path": "media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "9576179b6df8182760d0eb1da5898c09f6de5738679d4076e62a1041b64866bc", "size": 1153070, "path": "media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "622f2c07b65747a9a125eb3c04737fd439f5724a19a78bee5bdb2cf28ac6e37f", "size": 1159733, "path": "media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://a5ne5qm69w6avvfqwkaq2f30a426sgeggd8lwupnqnk766a6qzki1twusbbge50j8gba9l9z6ghlvmoufzuw7hr080ustrtj4pgmu894vmscbjhhiafx0w0dx4v6lxxi/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://9jw5s2n95fikdv1jythu95h34kd7ze6u7t5rwsu9jjqc1ei4lweqaot17abu1y1flistrtexl17s7hoaxdrolkevp6vnkyiojk9xmba1mdhiiw5izm94qfkmd8qyz2fy:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 5, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.43520843983992363, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.60740816591022, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.6842965185273651, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.6614799052153926, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "e32f206babc2825ce21715b938b57354f64231e089822aca554fad29505cf476", "size": 434, "artifact_path": "wandb-client-artifact://x2g1wzgda2hawa5gj0fw19v2ycdpczl0r5gu9awi2fluurwn5xt6celi6azy70d2cnfg60gv82euzgctcvncmdwa2si8imn8y0r960z63mavo4y64t10x56jr04shzek/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zizmmbl6dqd2dy71nkxcvxyvgazrgnluvsrn1510s7qkp0k8myu2w41jtl03pexhvl8d7ghw70bw4e7g2phtazuq00zia0gr00rnqe4qdla8dj1f8qm33k2sr6ulh8oh:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 256}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.43520843982696533, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.6074081659317017, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.6842964887619019, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.6614798903465271, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug-internal.log b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d2460a1aeebaa99df9838f7d8882352df8e97194 --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug-internal.log @@ -0,0 +1,760 @@ +2025-01-03 20:04:49,383 INFO StreamThr :1962513 [internal.py:wandb_internal():85] W&B internal server running at pid: 1962513, started at: 2025-01-03 20:04:49.383219 +2025-01-03 20:04:49,385 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:04:49,396 INFO WriterThread:1962513 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/run-z9oszy8k.wandb +2025-01-03 20:04:49,396 DEBUG SenderThread:1962513 [sender.py:send():379] send: header +2025-01-03 20:04:49,407 DEBUG SenderThread:1962513 [sender.py:send():379] send: run +2025-01-03 20:04:49,632 INFO SenderThread:1962513 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files +2025-01-03 20:04:49,632 INFO SenderThread:1962513 [sender.py:_start_run_threads():1188] run started: z9oszy8k with start time 1735952689.38404 +2025-01-03 20:04:49,646 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:04:49,646 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:04:49,718 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:04:49,723 DEBUG HandlerThread:1962513 [system_info.py:__init__():26] System info init +2025-01-03 20:04:49,723 DEBUG HandlerThread:1962513 [system_info.py:__init__():41] System info init done +2025-01-03 20:04:49,723 INFO HandlerThread:1962513 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:04:49,723 INFO SystemMonitor:1962513 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:04:49,724 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:04:49,724 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:04:49,724 INFO HandlerThread:1962513 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:04:49,724 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:04:49,725 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:04:49,725 INFO SystemMonitor:1962513 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:04:49,736 DEBUG HandlerThread:1962513 [system_info.py:probe():152] Probing system +2025-01-03 20:04:49,738 DEBUG HandlerThread:1962513 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:04:49,742 DEBUG HandlerThread:1962513 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:04:49,742 DEBUG HandlerThread:1962513 [system_info.py:probe():200] Probing system done +2025-01-03 20:04:49,743 DEBUG HandlerThread:1962513 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:04:49.736564', 'startedAt': '2025-01-04T01:04:49.376098', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3565.950875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.976, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4504.695, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 719.3927230834961}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:04:49,743 INFO HandlerThread:1962513 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:04:49,743 INFO HandlerThread:1962513 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:04:49,743 DEBUG HandlerThread:1962513 [system_info.py:_save_code():44] Saving code +2025-01-03 20:04:49,744 DEBUG HandlerThread:1962513 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:04:49,744 DEBUG HandlerThread:1962513 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:04:49,807 DEBUG HandlerThread:1962513 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:04:49,807 INFO HandlerThread:1962513 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:04:49,810 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:04:49,810 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:04:49,811 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:04:49,811 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:04:50,049 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:04:50,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:04:50,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:04:50,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:04:50,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:50,053 INFO HandlerThread:1962513 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/tb" + save: true +} +control { + relay_id: "7561cf02a9ee4f3f8920fa7c02089ede" +} +_info { + stream_id: "z9oszy8k" +} + +2025-01-03 20:04:50,053 DEBUG HandlerThread:1962513 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:04:50,077 INFO wandb-upload_1:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/as3goyvs-code/FISH/eval_robot.py +2025-01-03 20:04:50,078 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/spev4z88-wandb-metadata.json +2025-01-03 20:04:50,182 DEBUG SenderThread:1962513 [sender.py:send():379] send: telemetry +2025-01-03 20:04:50,182 DEBUG SenderThread:1962513 [sender.py:send():379] send: tbrecord +2025-01-03 20:04:50,182 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:04:50,183 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 with policy live +2025-01-03 20:04:50,389 INFO wandb-upload_1:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/qrn2cqly-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:04:50,633 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:04:50,633 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-metadata.json +2025-01-03 20:04:50,633 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/requirements.txt +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/diff.patch +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code/FISH/eval_robot.py +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code +2025-01-03 20:04:50,634 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code/FISH +2025-01-03 20:04:51,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:51,849 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/j6yptl3d-diff.patch +2025-01-03 20:04:52,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:52,632 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:04:53,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:54,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:55,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:04:55,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:56,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:57,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:58,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:04:59,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:00,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:00,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:01,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:02,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:03,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:03,524 DEBUG SenderThread:1962513 [sender.py:send():379] send: config +2025-01-03 20:05:04,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:04,636 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:05,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:05:05,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:05:05,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:05,131 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:06,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:06,636 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:07,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:08,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:08,637 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:09,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:10,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:10,638 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:10,962 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:11,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:12,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:12,638 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:13,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:14,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:14,639 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:15,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:16,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:16,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:16,640 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:17,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:18,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:18,640 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:19,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:20,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:05:20,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:05:20,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:21,055 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:21,140 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:21,641 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml +2025-01-03 20:05:22,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:23,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:24,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:25,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:26,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:26,642 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:27,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:27,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:28,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:29,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:30,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:31,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:32,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:33,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:33,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:34,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:35,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:05:35,050 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:05:35,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:36,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:37,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:38,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:38,205 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:39,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:40,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:41,058 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:42,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:43,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:44,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:44,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:45,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:45,375 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:05:45,376 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 with policy now +2025-01-03 20:05:45,376 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:05:45,378 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:05:45,379 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:05:45,380 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:05:45,625 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/nqi2tip8-media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 +2025-01-03 20:05:45,648 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:05:45,648 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 +2025-01-03 20:05:45,648 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:05:45,649 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos +2025-01-03 20:05:45,649 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media +2025-01-03 20:05:46,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:46,583 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/c75xshva-media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 +2025-01-03 20:05:46,648 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:47,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:48,059 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:49,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:49,092 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:49,726 DEBUG SystemMonitor:1962513 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:05:49,726 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:05:50,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:05:50,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:05:50,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:50,649 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:51,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:52,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:53,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:54,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:54,651 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:05:54,842 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:05:55,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:56,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:57,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:58,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:05:59,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:00,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:00,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:01,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:02,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:03,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:03,654 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:06:03,896 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/utc83uet-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:06:04,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:04,654 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:05,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:06:05,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:06:05,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:05,137 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:06,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:06,655 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:07,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:08,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:08,655 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:09,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:10,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:11,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:11,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:12,064 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:13,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:14,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:15,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:16,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:16,063 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:17,064 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:18,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:19,064 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:19,727 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:06:20,050 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:06:20,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:06:20,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:21,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:21,152 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:22,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:23,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:24,065 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:25,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:26,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:27,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:27,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:28,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:29,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:30,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:31,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:32,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:32,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:33,066 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:34,069 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:34,598 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:06:34,602 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:06:34,602 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 with policy now +2025-01-03 20:06:34,603 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:06:34,603 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:06:34,603 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:06:34,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:06:34,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:34,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 +2025-01-03 20:06:34,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:06:34,855 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/reahra3f-media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 +2025-01-03 20:06:35,052 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:06:35,052 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:06:35,071 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:35,143 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/7jb00nzn-media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 +2025-01-03 20:06:36,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:36,664 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:37,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:37,130 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:38,073 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:39,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:40,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:40,665 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:41,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:42,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:42,169 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:42,666 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:43,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:44,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:44,462 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:06:44,462 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:06:44,462 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:06:44,667 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:06:44,667 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:06:45,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:46,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:46,668 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:47,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:47,657 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:48,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:48,668 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:49,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:49,727 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:06:50,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:06:50,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:06:50,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:50,669 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:06:51,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:52,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:53,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:53,130 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:54,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:55,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:56,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:57,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:58,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:06:59,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:06:59,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:00,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:01,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:02,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:03,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:04,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:05,051 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:07:05,051 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:07:05,093 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:05,154 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:06,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:07,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:08,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:09,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:10,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:11,056 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:11,077 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:12,078 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:13,078 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:14,078 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:15,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:15,931 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:07:15,931 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 with policy now +2025-01-03 20:07:15,934 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:07:15,935 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:07:15,935 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:07:15,936 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:07:16,085 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:16,178 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/lhp9vxpr-media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 +2025-01-03 20:07:16,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:07:16,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:16,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 +2025-01-03 20:07:16,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:07:16,954 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/n6014vuj-media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 +2025-01-03 20:07:16,974 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:17,086 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:18,086 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:18,678 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:19,084 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:19,728 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:07:20,054 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:07:20,054 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:07:20,087 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:20,679 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:21,086 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:22,084 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:22,202 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:23,083 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:24,085 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:24,680 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:25,083 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:26,102 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:27,085 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:28,070 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:28,086 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:29,088 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:30,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:31,094 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:32,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:33,101 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:33,102 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:33,301 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:07:33,301 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:07:33,302 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:07:33,683 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:07:33,683 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:07:33,921 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/s2ld6lpz-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:07:34,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:34,684 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:35,062 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:07:35,062 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:07:35,105 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:36,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:36,684 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:37,105 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:38,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:38,308 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:38,685 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:07:39,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:40,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:41,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:42,096 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:43,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:44,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:44,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:45,097 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:46,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:47,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:48,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:49,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:49,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:49,728 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:07:50,053 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:07:50,054 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:07:50,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:51,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:52,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:53,098 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:54,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:54,222 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:07:55,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:56,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:57,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:58,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:07:59,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:00,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:00,099 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:01,100 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:02,100 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:03,100 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:04,118 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:05,060 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:08:05,060 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:08:05,109 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:05,141 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:05,341 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:08:05,341 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 with policy now +2025-01-03 20:08:05,342 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:08:05,342 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:08:05,343 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:08:05,344 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:08:05,614 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/j9wbowe1-media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 +2025-01-03 20:08:05,694 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:08:05,694 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 +2025-01-03 20:08:05,694 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:08:05,966 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/3v4wya38-media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 +2025-01-03 20:08:06,111 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:06,694 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:07,104 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:08,112 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:09,117 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:10,115 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:10,696 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:11,067 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:11,110 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:12,107 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:12,696 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:13,108 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:14,116 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:14,697 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:15,112 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:16,069 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:16,119 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:17,120 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:18,120 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:19,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:19,729 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:08:20,061 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:08:20,061 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:08:20,121 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:21,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:21,380 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:08:21,381 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:08:21,381 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:21,382 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:08:21,699 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:08:21,699 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:08:22,126 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:22,700 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:23,128 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:24,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:24,700 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:25,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:26,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:26,701 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:27,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:27,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:28,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:29,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:30,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:30,703 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:31,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:32,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:33,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:33,123 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:34,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:35,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:08:35,057 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:08:35,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:36,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:37,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:38,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:38,200 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:39,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:40,124 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:41,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:42,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:43,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:44,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:44,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:45,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:46,125 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:47,138 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:48,141 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:49,080 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:49,140 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:49,729 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:08:50,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:08:50,057 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:08:50,140 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:51,140 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:52,145 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:52,710 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:53,147 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:53,216 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:08:53,216 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 with policy now +2025-01-03 20:08:53,219 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:08:53,220 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:08:53,220 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:08:53,220 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:08:53,710 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:08:53,711 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 +2025-01-03 20:08:53,711 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval +2025-01-03 20:08:54,154 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:54,251 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:08:54,711 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:55,159 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:55,711 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:56,151 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:57,149 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:57,958 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/zgw01m04-media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 +2025-01-03 20:08:58,150 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:58,199 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/24jjo04m-media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 +2025-01-03 20:08:58,712 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:08:59,166 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:08:59,730 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:00,158 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:00,713 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:01,159 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:02,151 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:03,181 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:03,435 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:09:03,435 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:03,436 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:03,602 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:09:03,602 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media +2025-01-03 20:09:03,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table +2025-01-03 20:09:03,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval +2025-01-03 20:09:03,788 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:09:03,954 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/p0vddcs5-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:09:04,033 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/901biqc9-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:04,056 INFO wandb-upload_1:1962513 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdh9cniaz +2025-01-03 20:09:04,150 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:04,298 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:09:04,308 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:09:04,511 INFO SenderThread:1962513 [sender.py:send_request_log_artifact():1525] logged artifact run-z9oszy8k-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgyMzkxOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2NzM5OQ==', 'latestArtifact': None}} +2025-01-03 20:09:04,511 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:09:04,511 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 20:09:04,512 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:09:04,512 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,512 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,512 DEBUG SenderThread:1962513 [sender.py:send():379] send: telemetry +2025-01-03 20:09:04,512 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,513 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,513 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,513 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,515 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,515 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,516 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,516 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,516 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,517 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,517 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,517 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,517 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,517 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,518 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,518 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,518 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,518 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,519 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,519 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,519 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:04,519 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:04,520 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:09:04,714 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:04,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:04,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:04,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:09:04,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval +2025-01-03 20:09:04,737 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/0vqfzc2f-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:09:04,859 INFO wandb-upload_1:1962513 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppere7wef +2025-01-03 20:09:04,885 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/k2u43mkq-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:09:05,057 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:09:05,150 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:05,279 INFO SenderThread:1962513 [sender.py:send_request_log_artifact():1525] logged artifact run-z9oszy8k-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgyMzkzMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2NzQwMg==', 'latestArtifact': None}} +2025-01-03 20:09:05,279 DEBUG SenderThread:1962513 [sender.py:send():379] send: files +2025-01-03 20:09:05,279 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json with policy now +2025-01-03 20:09:05,279 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:09:05,279 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:05,280 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:05,280 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:05,280 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:09:05,514 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/r5golrpm-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:09:05,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:06,073 DEBUG SenderThread:1962513 [sender.py:send():379] send: exit +2025-01-03 20:09:06,073 INFO SenderThread:1962513 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:send_exit():588] handling runtime: 256 +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:send_exit():594] send defer +2025-01-03 20:09:06,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,074 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:09:06,074 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:09:06,074 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:09:06,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,075 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:09:06,075 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:06,075 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:09:06,075 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:09:06,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,075 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:09:06,075 INFO HandlerThread:1962513 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:09:06,075 DEBUG SystemMonitor:1962513 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:09:06,075 DEBUG SystemMonitor:1962513 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:09:06,076 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:09:06,076 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:09:06,147 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:09:06,148 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:09:06,148 INFO HandlerThread:1962513 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:09:06,148 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:06,148 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:09:06,148 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:09:06,148 DEBUG SenderThread:1962513 [sender.py:send():379] send: stats +2025-01-03 20:09:06,149 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,149 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:09:06,149 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:06,149 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:09:06,149 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:09:06,149 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:06,149 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:09:06,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:06,715 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:08,717 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:11,718 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml +2025-01-03 20:09:12,441 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:12,441 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:12,441 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:09:12,442 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:12,442 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:09:12,442 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:09:12,442 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:12,443 DEBUG SenderThread:1962513 [sender.py:send():379] send: history +2025-01-03 20:09:12,443 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:12,443 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:09:12,443 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:09:12,444 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:12,444 DEBUG SenderThread:1962513 [sender.py:send():379] send: summary +2025-01-03 20:09:12,444 INFO SenderThread:1962513 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:09:12,444 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:12,444 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:09:12,444 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:09:12,444 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:12,444 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:09:12,445 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:12,445 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:09:12,445 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:09:12,445 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:09:12,445 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:12,445 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:09:12,445 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:12,445 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:09:12,719 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:13,074 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:13,175 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:09:13,175 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:13,176 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:13,176 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:09:13,176 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:13,176 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:09:13,176 INFO SenderThread:1962513 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:09:13,178 INFO SenderThread:1962513 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:09:13,180 INFO SenderThread:1962513 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:09:13,191 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:09:13,191 DEBUG SenderThread:1962513 [sender.py:send():379] send: artifact +2025-01-03 20:09:13,194 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:13,194 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:09:13,664 INFO wandb-upload_2:1962513 [upload_job.py:push():85] Skipped uploading /tmp/tmpid9omv5h/wandb-job.json +2025-01-03 20:09:13,664 INFO wandb-upload_1:1962513 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr5ulhnd_ +2025-01-03 20:09:13,719 INFO Thread-12 :1962513 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:13,790 INFO wandb-upload_0:1962513 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpg0eiejcp +2025-01-03 20:09:14,075 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:14,296 INFO SenderThread:1962513 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzgyNDA5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzgxMTc2Nw==', 'versionIndex': 405}}} +2025-01-03 20:09:14,296 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:14,296 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:09:14,296 INFO SenderThread:1962513 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:09:14,720 INFO SenderThread:1962513 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files +2025-01-03 20:09:14,720 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml config.yaml +2025-01-03 20:09:14,720 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json wandb-summary.json +2025-01-03 20:09:14,720 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/requirements.txt requirements.txt +2025-01-03 20:09:14,722 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:09:14,722 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:14,722 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log output.log +2025-01-03 20:09:14,723 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/diff.patch diff.patch +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 media/videos/eval/0_eval_0_192a2c0c13410dee2aa9.mp4 +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 media/videos/eval/1_eval_1_a412d87eb0226343aaae.mp4 +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 media/videos/eval/3_eval_3_9576179b6df8182760d0.mp4 +2025-01-03 20:09:14,726 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 media/videos/eval/2_eval_2_83a3bb9782b59fb8ba2c.mp4 +2025-01-03 20:09:14,727 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 media/videos/eval/4_eval_4_622f2c07b65747a9a125.mp4 +2025-01-03 20:09:14,727 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:09:14,727 INFO SenderThread:1962513 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:09:14,727 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:09:14,727 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:14,727 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:14,729 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:09:14,729 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:14,729 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:09:14,729 INFO SenderThread:1962513 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:09:14,896 INFO wandb-upload_2:1962513 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/config.yaml +2025-01-03 20:09:14,935 INFO wandb-upload_1:1962513 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/wandb-summary.json +2025-01-03 20:09:14,935 INFO wandb-upload_0:1962513 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/requirements.txt +2025-01-03 20:09:14,952 INFO wandb-upload_4:1962513 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/files/output.log +2025-01-03 20:09:15,019 INFO wandb-upload_3:1962513 [upload_job.py:push():130] Uploaded file /tmp/tmpxuf7mkr4wandb/w688wnoc-events.out.tfevents.1735952690.leonmkim-ROG-Strix-G15CS-G15CS.1962348.0 +2025-01-03 20:09:15,076 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:15,076 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:15,220 INFO Thread-11 (_thread_body):1962513 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:09:15,220 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:15,220 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:09:15,220 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:15,220 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:09:15,220 INFO SenderThread:1962513 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:09:15,221 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:09:15,222 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:15,222 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:09:15,222 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:15,222 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:09:15,222 INFO SenderThread:1962513 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:09:15,286 INFO SenderThread:1962513 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:09:15,286 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:09:15,286 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:15,286 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:09:15,286 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:15,286 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:09:15,287 INFO SenderThread:1962513 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:09:15,287 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:09:15,287 INFO HandlerThread:1962513 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:09:15,287 DEBUG SenderThread:1962513 [sender.py:send():379] send: final +2025-01-03 20:09:15,287 DEBUG SenderThread:1962513 [sender.py:send():379] send: footer +2025-01-03 20:09:15,287 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: defer +2025-01-03 20:09:15,287 INFO SenderThread:1962513 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:09:15,288 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:15,288 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:09:15,288 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:09:15,288 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:15,288 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:09:15,289 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:09:15,289 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:09:15,291 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:09:15,349 DEBUG SenderThread:1962513 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:09:15,350 INFO MainThread:1962513 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:09:15,350 INFO MainThread:1962513 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:09:15,350 INFO MainThread:1962513 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:09:15,351 DEBUG HandlerThread:1962513 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:09:15,351 INFO HandlerThread:1962513 [handler.py:finish():882] shutting down handler +2025-01-03 20:09:16,288 INFO WriterThread:1962513 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/run-z9oszy8k.wandb +2025-01-03 20:09:16,350 INFO SenderThread:1962513 [sender.py:finish():1615] shutting down sender +2025-01-03 20:09:16,350 INFO SenderThread:1962513 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:09:16,350 INFO SenderThread:1962513 [file_pusher.py:join():175] waiting for file pusher diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug.log b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ce1ef2ffe5a2e60310c23898c851ec238624c70c --- /dev/null +++ b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Configure stats pid to 1962348 +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/settings +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug.log +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/logs/debug-internal.log +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k'} +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:init():619] starting backend +2025-01-03 20:04:49,378 INFO MainThread:1962348 [wandb_init.py:init():623] setting up manager +2025-01-03 20:04:49,382 INFO MainThread:1962348 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:04:49,383 INFO MainThread:1962348 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:04:49,395 INFO MainThread:1962348 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:04:49,405 INFO MainThread:1962348 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:04:49,646 INFO MainThread:1962348 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:04:49,714 INFO MainThread:1962348 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:04:49,714 INFO MainThread:1962348 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:04:50,051 INFO MainThread:1962348 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:04:50,051 INFO MainThread:1962348 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:04:50,051 INFO MainThread:1962348 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:04:50,051 INFO MainThread:1962348 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:04:50,052 INFO MainThread:1962348 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:04:50,052 INFO MainThread:1962348 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/z9oszy8k/tb, True +2025-01-03 20:05:03,524 INFO MainThread:1962348 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:09:16,351 WARNING MsgRouterThr:1962348 [router.py:message_loop():77] message_loop has been closed diff --git a/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/run-z9oszy8k.wandb b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/run-z9oszy8k.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7f98f0458fe03c2e15fb23d05b00d5fc04f06bb2 Binary files /dev/null and b/z9oszy8k/wandb/run-20250103_200449-z9oszy8k/run-z9oszy8k.wandb differ