diff --git a/.gitattributes b/.gitattributes index 40e859be2deb8ae933dbf032e6c4d6b25d6987df..d8c6e6d2b35e7e2c8cbcaf6769dfff05e22a6c4d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -106,3 +106,76 @@ mo0b6mh9/episode_rosbags/episode_18_2025-01-18-20-10-13.bag filter=lfs diff=lfs 138tidw7/episode_rosbags/episode_4_2025-01-18-19-34-29.bag filter=lfs diff=lfs merge=lfs -text mo0b6mh9/episode_rosbags/episode_13_2025-01-18-20-06-00.bag filter=lfs diff=lfs merge=lfs -text mo0b6mh9/episode_rosbags/episode_15_2025-01-18-20-07-44.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_0_2025-01-18-19-31-21.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_9_2025-01-18-20-02-21.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_5_2025-01-18-19-59-26.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/run-mo0b6mh9.wandb filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_0_2025-01-18-19-55-45.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_17_2025-01-18-20-09-31.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_2_2025-01-18-19-57-07.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_12_2025-01-18-20-04-42.bag filter=lfs diff=lfs merge=lfs -text +2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 filter=lfs diff=lfs merge=lfs -text +2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 filter=lfs diff=lfs merge=lfs -text +2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 filter=lfs diff=lfs merge=lfs -text +2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 filter=lfs diff=lfs merge=lfs -text +2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_8_2025-01-18-20-01-38.bag filter=lfs diff=lfs merge=lfs -text +2o1us5j0/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2o1us5j0/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2o1us5j0/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2o1us5j0/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2o1us5j0/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_16_2025-01-18-20-08-36.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_4_2025-01-18-19-58-35.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_11_2025-01-18-20-03-53.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_7_2025-01-18-19-36-44.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_8_2025-01-18-19-37-30.bag filter=lfs diff=lfs merge=lfs -text +2o1us5j0/episode_rosbags/episode_4_2025-01-21-21-17-17.bag filter=lfs diff=lfs merge=lfs -text +2o1us5j0/episode_rosbags/episode_3_2025-01-21-21-16-28.bag filter=lfs diff=lfs merge=lfs -text +2o1us5j0/episode_rosbags/episode_2_2025-01-21-21-15-47.bag filter=lfs diff=lfs merge=lfs -text +2o1us5j0/episode_rosbags/episode_1_2025-01-21-21-15-04.bag filter=lfs diff=lfs merge=lfs -text +2o1us5j0/episode_rosbags/episode_0_2025-01-21-21-14-15.bag filter=lfs diff=lfs merge=lfs -text +q1ti958r/episode_rosbags/episode_10_2025-01-18-18-57-52.bag filter=lfs diff=lfs merge=lfs -text +q1ti958r/episode_rosbags/episode_0_2025-01-18-18-50-38.bag filter=lfs diff=lfs merge=lfs -text +q1ti958r/episode_rosbags/episode_11_2025-01-18-18-58-35.bag filter=lfs diff=lfs merge=lfs -text +q1ti958r/episode_rosbags/episode_19_2025-01-18-19-05-54.bag filter=lfs diff=lfs merge=lfs -text diff --git a/138tidw7/episode_rosbags/episode_0_2025-01-18-19-31-21.bag b/138tidw7/episode_rosbags/episode_0_2025-01-18-19-31-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..8bbf5c25374a4d3a45fced99a1fa7c63668130dc --- /dev/null +++ b/138tidw7/episode_rosbags/episode_0_2025-01-18-19-31-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0345118b7f11b561345d996b0cbaddce3512fe8f904bc77f3d272512cd47aa36 +size 1412992294 diff --git a/138tidw7/episode_rosbags/episode_7_2025-01-18-19-36-44.bag b/138tidw7/episode_rosbags/episode_7_2025-01-18-19-36-44.bag new file mode 100644 index 0000000000000000000000000000000000000000..c943c73aa21537158196c33cba5e43d2eaa34281 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_7_2025-01-18-19-36-44.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f711583c68fb780917ffc116c0d50d7effc09c42a2914528698283d3a211620 +size 1411419333 diff --git a/138tidw7/episode_rosbags/episode_8_2025-01-18-19-37-30.bag b/138tidw7/episode_rosbags/episode_8_2025-01-18-19-37-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..f1a1d006530e116f770232df199121cfa6bd5e83 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_8_2025-01-18-19-37-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:279cc8ba9fc1ab8e2348d4e1d1a299017ee5403fa7cc34f9f4180c921cd7c677 +size 1409430383 diff --git a/2o1us5j0/.hydra/config.yaml b/2o1us5j0/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f829134d57914397f938447dc055c8c1721578e --- /dev/null +++ b/2o1us5j0/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/2o1us5j0/.hydra/hydra.yaml b/2o1us5j0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a7ebe74eafe36e950078108b54284bc0e6d27dd --- /dev/null +++ b/2o1us5j0/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/2o1us5j0/.hydra/overrides.yaml b/2o1us5j0/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/2o1us5j0/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/2o1us5j0/episode_rosbags/aligned_depth_to_color_K.npy b/2o1us5j0/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/2o1us5j0/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/2o1us5j0/episode_rosbags/cam_tf_world.npy b/2o1us5j0/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/2o1us5j0/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/2o1us5j0/episode_rosbags/color_K.npy b/2o1us5j0/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/2o1us5j0/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/2o1us5j0/episode_rosbags/depth_K.npy b/2o1us5j0/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/2o1us5j0/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/2o1us5j0/episode_rosbags/episode_0_2025-01-21-21-14-15.bag b/2o1us5j0/episode_rosbags/episode_0_2025-01-21-21-14-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..3956bab08e5b54c69363613e9361b47e398fb584 --- /dev/null +++ b/2o1us5j0/episode_rosbags/episode_0_2025-01-21-21-14-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57a0c20f908cd31691bc24205afda0650085e57132a13829dc184002daa64600 +size 1411064067 diff --git a/2o1us5j0/episode_rosbags/episode_1_2025-01-21-21-15-04.bag b/2o1us5j0/episode_rosbags/episode_1_2025-01-21-21-15-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..a859ebaa78e1f3d58485584dae4c50475bf2c514 --- /dev/null +++ b/2o1us5j0/episode_rosbags/episode_1_2025-01-21-21-15-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11f356c3fee9b2a1d86906ef94c5f9e21a061b3f57fdefc15db40b36b2a30e4a +size 1413258865 diff --git a/2o1us5j0/episode_rosbags/episode_2_2025-01-21-21-15-47.bag b/2o1us5j0/episode_rosbags/episode_2_2025-01-21-21-15-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..f1d2d03916fde65a1449c9c85133da9bf6921eb9 --- /dev/null +++ b/2o1us5j0/episode_rosbags/episode_2_2025-01-21-21-15-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:692709e950366e7ff9b26c765297178fd3777e401351cd884b066742fc56b1e5 +size 1421361186 diff --git a/2o1us5j0/episode_rosbags/episode_3_2025-01-21-21-16-28.bag b/2o1us5j0/episode_rosbags/episode_3_2025-01-21-21-16-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..77ed5e93116dc1d989cbfa814973d38954b5a930 --- /dev/null +++ b/2o1us5j0/episode_rosbags/episode_3_2025-01-21-21-16-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8990b0f7049d9d7c2ec85b574362e07f7b7f7c294819cd973514f60502d3103 +size 1409234017 diff --git a/2o1us5j0/episode_rosbags/episode_4_2025-01-21-21-17-17.bag b/2o1us5j0/episode_rosbags/episode_4_2025-01-21-21-17-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..97f79d488fdb92878cd52ab944fed3426b92d3b9 --- /dev/null +++ b/2o1us5j0/episode_rosbags/episode_4_2025-01-21-21-17-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:365dfe817c04adcaeb0f2c1ff8fe14d240386980d795c753a7445c0fe5364c9d +size 1407795823 diff --git a/2o1us5j0/eval_robot.log b/2o1us5j0/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..b3e01ed3b58b869ddb828398428ce379f48e3c52 --- /dev/null +++ b/2o1us5j0/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:13:33,402][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:13:33,404][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/2o1us5j0/eval_video/0_eval.mp4 b/2o1us5j0/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..52650538389a648a9da140f02e7bc0a9fc82408f --- /dev/null +++ b/2o1us5j0/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b23de2781f443affd89f3f16a927cac8f5165c1720fe36a6bb9c646d5d4dbdb +size 1174690 diff --git a/2o1us5j0/eval_video/1_eval.mp4 b/2o1us5j0/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d724de7b4b49a1dc7aa79970a801ddb0322e3d5a --- /dev/null +++ b/2o1us5j0/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27ab141d29c0759686c9c2f7d491f49f02a175914b83f78068ff3dd45e2c9bbb +size 1174970 diff --git a/2o1us5j0/eval_video/2_eval.mp4 b/2o1us5j0/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92e8de94c4dc9dafa122c80b75bdd89db7142e08 --- /dev/null +++ b/2o1us5j0/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02193a80ccc515d7b53d93235e42189751e8721c817731b26f7a96a753a9879d +size 1185158 diff --git a/2o1us5j0/eval_video/3_eval.mp4 b/2o1us5j0/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2447900cfd631d470171cd4e7e4ac81f1cbe1730 --- /dev/null +++ b/2o1us5j0/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1146ecf321ca00c51feeeb4e45480b601f9ffa7208012261f29f436620d8a4d +size 1214627 diff --git a/2o1us5j0/eval_video/4_eval.mp4 b/2o1us5j0/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..42b1d00283e87fcee0fab67ec3bc4357c0f2ab79 --- /dev/null +++ b/2o1us5j0/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c129977338f91caa371f496f88f6b0d2c4079eb1b71d2b5df643d085a7e841 +size 1195894 diff --git a/2o1us5j0/tb/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 b/2o1us5j0/tb/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 new file mode 100644 index 0000000000000000000000000000000000000000..fc742e798b3195aface3a8258e46283b62a912bc --- /dev/null +++ b/2o1us5j0/tb/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40de8072ee9739cadc2ff06212acfb418e02571679e97f8e91771f1e8cea6a63 +size 6539 diff --git a/2o1us5j0/wandb/debug-internal.log b/2o1us5j0/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5da1ba763ce361a0dc9983a0cec92f81b6eb5b3c --- /dev/null +++ b/2o1us5j0/wandb/debug-internal.log @@ -0,0 +1,785 @@ +2025-01-21 21:13:39,120 INFO StreamThr :2647496 [internal.py:wandb_internal():85] W&B internal server running at pid: 2647496, started at: 2025-01-21 21:13:39.120468 +2025-01-21 21:13:39,123 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:13:39,133 INFO WriterThread:2647496 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/run-2o1us5j0.wandb +2025-01-21 21:13:39,133 DEBUG SenderThread:2647496 [sender.py:send():379] send: header +2025-01-21 21:13:39,141 DEBUG SenderThread:2647496 [sender.py:send():379] send: run +2025-01-21 21:13:39,450 INFO SenderThread:2647496 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files +2025-01-21 21:13:39,450 INFO SenderThread:2647496 [sender.py:_start_run_threads():1188] run started: 2o1us5j0 with start time 1737512019.120725 +2025-01-21 21:13:39,462 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:13:39,462 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:13:39,541 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:13:39,546 DEBUG HandlerThread:2647496 [system_info.py:__init__():26] System info init +2025-01-21 21:13:39,546 DEBUG HandlerThread:2647496 [system_info.py:__init__():41] System info init done +2025-01-21 21:13:39,546 INFO HandlerThread:2647496 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:13:39,546 INFO SystemMonitor:2647496 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:13:39,547 INFO HandlerThread:2647496 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:13:39,547 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:13:39,548 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:13:39,549 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:13:39,549 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:13:39,549 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:13:39,560 DEBUG HandlerThread:2647496 [system_info.py:probe():152] Probing system +2025-01-21 21:13:39,561 DEBUG HandlerThread:2647496 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:13:39,566 DEBUG HandlerThread:2647496 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:13:39,566 DEBUG HandlerThread:2647496 [system_info.py:probe():200] Probing system done +2025-01-21 21:13:39,566 DEBUG HandlerThread:2647496 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:13:39.560261', 'startedAt': '2025-01-22T02:13:39.112325', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.98925, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4491.036, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 350.6014938354492}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:13:39,566 INFO HandlerThread:2647496 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:13:39,566 INFO HandlerThread:2647496 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:13:39,566 DEBUG HandlerThread:2647496 [system_info.py:_save_code():44] Saving code +2025-01-21 21:13:39,567 DEBUG HandlerThread:2647496 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:13:39,567 DEBUG HandlerThread:2647496 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:13:39,629 DEBUG HandlerThread:2647496 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:13:39,630 INFO HandlerThread:2647496 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:13:39,633 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:13:39,633 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:13:39,633 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:13:39,633 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:13:39,897 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:13:39,897 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:13:39,899 DEBUG SenderThread:2647496 [sender.py:send():379] send: telemetry +2025-01-21 21:13:39,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:13:39,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:39,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:13:39,902 INFO HandlerThread:2647496 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/tb" + save: true +} +control { + relay_id: "07530a4f366b4424bd90a669e2ceac72" +} +_info { + stream_id: "2o1us5j0" +} + +2025-01-21 21:13:39,902 DEBUG HandlerThread:2647496 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:13:39,926 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/fxfagu85-diff.patch +2025-01-21 21:13:39,928 INFO wandb-upload_1:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/yvoo1sr9-code/FISH/eval_robot.py +2025-01-21 21:13:39,938 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/rtiscy5w-wandb-metadata.json +2025-01-21 21:13:40,029 DEBUG SenderThread:2647496 [sender.py:send():379] send: tbrecord +2025-01-21 21:13:40,029 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:13:40,029 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 with policy live +2025-01-21 21:13:40,212 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/6be263eg-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/diff.patch +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-metadata.json +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/requirements.txt +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code/FISH/eval_robot.py +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code/FISH +2025-01-21 21:13:40,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:41,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:42,451 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:42,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:43,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:44,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:13:44,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:45,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:46,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:47,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:48,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:49,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:13:49,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:50,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:51,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:52,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:53,110 DEBUG SenderThread:2647496 [sender.py:send():379] send: config +2025-01-21 21:13:53,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:54,454 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:54,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:13:54,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:13:54,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:54,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:13:55,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:56,455 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:56,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:57,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:58,455 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:58,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:59,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:00,456 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:00,547 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:00,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:01,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:02,457 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:02,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:03,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:04,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:05,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:05,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:06,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:07,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:08,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:09,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:14:09,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:14:09,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:10,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:11,004 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:11,459 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml +2025-01-21 21:14:11,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:12,459 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:12,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:13,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:14,460 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:14,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:15,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:16,327 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:16,461 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:16,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:17,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:18,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:19,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:20,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:21,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:21,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:22,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:23,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:24,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:14:24,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:14:24,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:25,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:26,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:26,988 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:27,905 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:28,905 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:29,905 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:30,905 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:31,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:32,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:32,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:33,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:34,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:35,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:36,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:37,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:38,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:38,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:39,550 DEBUG SystemMonitor:2647496 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:14:39,550 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:14:39,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:14:39,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:14:39,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:40,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:41,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:42,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:43,049 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:14:43,050 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:14:43,050 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 with policy now +2025-01-21 21:14:43,050 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:14:43,051 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:14:43,051 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:14:43,299 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/nhde7sj8-media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 +2025-01-21 21:14:43,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:14:43,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 +2025-01-21 21:14:43,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos +2025-01-21 21:14:43,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:14:43,470 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media +2025-01-21 21:14:43,767 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/rhq817hg-media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 +2025-01-21 21:14:43,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:44,051 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:44,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:44,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:45,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:46,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:47,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:48,471 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:48,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:49,346 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:49,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:50,471 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:50,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:51,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:52,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:53,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:54,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:14:54,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:14:54,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:55,033 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:55,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:56,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:57,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:58,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:59,911 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:00,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:00,910 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:01,475 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:15:01,733 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/77lcrimn-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:15:01,910 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:02,475 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:02,911 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:03,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:04,476 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:04,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:05,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:06,156 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:06,477 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:06,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:07,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:08,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:09,551 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:15:09,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:15:09,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:15:09,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:10,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:11,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:12,035 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:12,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:13,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:14,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:15,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:16,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:17,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:17,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:18,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:19,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:20,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:21,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:22,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:22,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:23,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:24,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:15:24,899 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:15:24,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:25,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:26,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:27,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:27,990 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:28,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:29,915 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:30,915 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:31,915 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:32,915 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:33,107 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:33,153 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:15:33,154 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:15:33,154 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 with policy now +2025-01-21 21:15:33,154 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:15:33,154 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:15:33,157 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:15:33,395 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/16b80rlp-media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 +2025-01-21 21:15:33,485 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:15:33,486 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 +2025-01-21 21:15:33,486 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:15:33,785 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/z7rklwx6-media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 +2025-01-21 21:15:33,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:34,486 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:34,919 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:35,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:36,919 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:37,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:38,487 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:38,789 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:38,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:39,551 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:15:39,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:15:39,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:15:39,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:40,488 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:40,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:41,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:42,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:43,489 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:15:43,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:44,333 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:44,339 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:15:44,339 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:15:44,339 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:15:44,489 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:15:44,490 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:44,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:45,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:46,490 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:46,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:47,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:48,491 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:48,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:49,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:49,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:50,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:51,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:52,921 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:53,921 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:54,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:15:54,899 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:15:54,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:54,973 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:55,927 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:56,927 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:57,927 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:58,927 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:59,928 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:00,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:00,929 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:01,929 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:02,496 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:02,930 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:03,930 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:04,930 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:05,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:05,930 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:06,931 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:07,931 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:08,497 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:08,931 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:09,552 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:16:09,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:16:09,899 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:16:09,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:10,936 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:11,011 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:11,937 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:12,937 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:13,937 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:14,946 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:15,329 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:16:15,329 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 with policy now +2025-01-21 21:16:15,334 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:16:15,334 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:16:15,334 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:16:15,334 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:16:15,500 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:16:15,500 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 +2025-01-21 21:16:15,500 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:16:15,566 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/bhs56rj6-media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 +2025-01-21 21:16:15,839 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/xtywq2ny-media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 +2025-01-21 21:16:15,946 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:16,373 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:16,500 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:16,946 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:17,942 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:18,942 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:19,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:20,501 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:20,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:21,767 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:21,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:22,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:23,948 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:24,503 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:24,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:16:24,902 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:16:24,953 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:25,156 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:16:25,156 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:16:25,156 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:16:25,503 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:16:25,503 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:16:25,767 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/eblg8jeh-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:16:25,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:26,503 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:26,869 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:26,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:27,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:28,504 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:28,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:29,948 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:30,505 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:30,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:31,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:32,107 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:32,506 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:32,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:33,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:34,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:35,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:36,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:37,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:37,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:38,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:39,552 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:16:39,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:16:39,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:16:39,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:40,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:41,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:42,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:42,981 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:43,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:44,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:45,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:46,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:47,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:48,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:48,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:49,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:50,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:51,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:52,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:53,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:53,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:54,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:16:54,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:16:54,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:55,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:56,953 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:57,954 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:58,050 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:16:58,050 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 with policy now +2025-01-21 21:16:58,053 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:16:58,053 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:16:58,053 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:16:58,053 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:16:58,299 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/vpsz4ygw-media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 +2025-01-21 21:16:58,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:16:58,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:58,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 +2025-01-21 21:16:58,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:16:58,801 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/eqn0vb4m-media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 +2025-01-21 21:16:58,955 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:59,080 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:59,954 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:00,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:00,958 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:01,966 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:02,516 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:02,962 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:03,960 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:04,470 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:04,954 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:05,956 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:06,517 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:06,960 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:07,964 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:08,966 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:09,553 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:17:09,553 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:09,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:17:09,907 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:17:09,967 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:10,965 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:11,973 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:12,973 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:13,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:14,211 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:17:14,211 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:14,212 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:14,520 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:17:14,520 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:17:14,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:14,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:15,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:16,521 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:16,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:18,017 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:18,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:19,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:20,522 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:20,911 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:20,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:21,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:22,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:23,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:24,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:17:24,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:17:24,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:25,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:26,061 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:26,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:27,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:28,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:29,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:30,525 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:30,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:31,911 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:31,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:32,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:33,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:34,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:35,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:36,527 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:36,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:36,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:37,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:38,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:39,553 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:17:39,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:17:39,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:17:39,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:40,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:41,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:42,041 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:42,530 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:42,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:43,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:44,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:45,975 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:46,326 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:17:46,327 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 with policy now +2025-01-21 21:17:46,333 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:17:46,333 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:17:46,333 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:46,334 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:46,531 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:17:46,531 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:46,531 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 +2025-01-21 21:17:46,531 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:17:46,555 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/i1jwc2vc-media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 +2025-01-21 21:17:46,824 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/mrdlrkek-media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 +2025-01-21 21:17:46,978 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:47,350 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:47,976 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:48,532 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:48,978 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:50,005 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:50,983 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:51,982 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:52,533 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:52,807 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:52,982 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:53,980 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:54,534 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:54,926 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:17:54,926 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:17:54,994 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:55,992 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:56,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:57,535 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:17:57,798 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/c9psj602-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:17:57,995 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:58,267 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:17:58,267 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:58,267 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:58,268 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:58,342 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:17:58,342 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:17:58,535 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval +2025-01-21 21:17:58,549 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:17:58,730 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/3cpmjebx-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:17:58,825 INFO wandb-upload_0:2647496 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3ejflfr6 +2025-01-21 21:17:58,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:17:58,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:17:58,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:59,232 INFO SenderThread:2647496 [sender.py:send_request_log_artifact():1525] logged artifact run-2o1us5j0-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY1ODIxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5Mzc4MA==', 'latestArtifact': None}} +2025-01-21 21:17:59,233 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:17:59,233 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-21 21:17:59,233 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:17:59,233 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,234 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,234 DEBUG SenderThread:2647496 [sender.py:send():379] send: telemetry +2025-01-21 21:17:59,234 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,234 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,234 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,235 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,235 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,237 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,237 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,237 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,237 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,238 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,238 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,238 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,238 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,239 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,239 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,239 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,239 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,240 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,240 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,240 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,240 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,240 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,241 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:17:59,495 INFO wandb-upload_1:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/mvi0bfk2-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:17:59,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:17:59,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:17:59,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json +2025-01-21 21:17:59,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval +2025-01-21 21:17:59,587 INFO wandb-upload_0:2647496 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjo3psi_s +2025-01-21 21:17:59,718 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/u5jhwjet-media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json +2025-01-21 21:17:59,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:00,109 INFO SenderThread:2647496 [sender.py:send_request_log_artifact():1525] logged artifact run-2o1us5j0-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY1ODIyOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5Mzc4Ng==', 'latestArtifact': None}} +2025-01-21 21:18:00,109 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:18:00,109 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json with policy now +2025-01-21 21:18:00,109 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:18:00,109 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:18:00,110 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:18:00,366 INFO wandb-upload_1:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/hkyl84mn-media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json +2025-01-21 21:18:00,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:18:00,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:18:00,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:01,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:02,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:03,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:04,111 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:04,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:05,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:06,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:07,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:08,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:09,554 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:18:09,555 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:09,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:18:09,901 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:18:09,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:10,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:11,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:12,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:13,986 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:14,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:14,986 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:15,986 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:16,901 DEBUG SenderThread:2647496 [sender.py:send():379] send: exit +2025-01-21 21:18:16,902 INFO SenderThread:2647496 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:18:16,902 INFO SenderThread:2647496 [sender.py:send_exit():588] handling runtime: 277 +2025-01-21 21:18:16,902 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:send_exit():594] send defer +2025-01-21 21:18:16,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,903 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:18:16,903 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:18:16,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,903 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:18:16,903 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:18:16,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,904 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:18:16,904 INFO HandlerThread:2647496 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:18:16,904 DEBUG SystemMonitor:2647496 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:18:16,904 DEBUG SystemMonitor:2647496 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:18:16,904 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:18:16,905 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:18:16,957 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:18:16,957 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:18:16,958 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:18:16,958 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:16,958 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:18:16,958 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:18:16,958 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:18:16,959 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,959 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:18:16,959 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:16,959 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:18:16,959 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:18:16,960 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,960 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:18:17,543 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:18:18,544 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:18:23,274 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:23,274 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:23,274 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:18:23,276 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:23,276 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:18:23,276 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:18:23,276 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:23,276 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:18:23,276 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:18:23,276 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:23,277 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:18:23,278 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:18:23,278 DEBUG SenderThread:2647496 [sender.py:send():379] send: summary +2025-01-21 21:18:23,282 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:18:23,282 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:23,282 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:18:23,282 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:18:23,282 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:23,282 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:18:23,282 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:23,282 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:18:23,286 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:23,549 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:18:23,549 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:23,550 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:18:23,550 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:23,550 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:18:23,550 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:18:23,550 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml +2025-01-21 21:18:23,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:24,357 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:18:24,357 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:24,357 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:24,358 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:18:24,358 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:24,358 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:18:24,358 INFO SenderThread:2647496 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:18:24,360 INFO SenderThread:2647496 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:18:24,363 INFO SenderThread:2647496 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:18:24,373 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:18:24,373 DEBUG SenderThread:2647496 [sender.py:send():379] send: artifact +2025-01-21 21:18:24,373 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:24,376 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:18:24,551 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:18:24,565 INFO SenderThread:2647496 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:18:24,565 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:24,565 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:18:24,565 INFO SenderThread:2647496 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:18:24,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml config.yaml +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json wandb-summary.json +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/requirements.txt requirements.txt +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log output.log +2025-01-21 21:18:25,557 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/diff.patch diff.patch +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:18:25,559 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:25,559 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:25,559 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:18:25,559 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:25,559 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:18:25,559 INFO SenderThread:2647496 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:18:25,746 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml +2025-01-21 21:18:25,758 INFO wandb-upload_1:2647496 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/requirements.txt +2025-01-21 21:18:25,774 INFO wandb-upload_4:2647496 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:18:25,775 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:18:25,884 INFO wandb-upload_3:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/k7rh9v1y-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:18:25,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:25,903 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:26,085 INFO Thread-11 (_thread_body):2647496 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:18:26,085 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:26,085 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:18:26,085 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:26,085 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:18:26,085 INFO SenderThread:2647496 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:18:26,087 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:18:26,087 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:26,087 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:18:26,087 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:26,087 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:18:26,087 INFO SenderThread:2647496 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:18:26,295 INFO SenderThread:2647496 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:18:26,295 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:18:26,296 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:26,296 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:18:26,296 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:26,296 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:18:26,296 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:18:26,296 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:26,296 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:18:26,296 DEBUG SenderThread:2647496 [sender.py:send():379] send: final +2025-01-21 21:18:26,296 DEBUG SenderThread:2647496 [sender.py:send():379] send: footer +2025-01-21 21:18:26,297 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:26,297 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:18:26,297 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:26,297 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:18:26,297 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:26,298 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:26,298 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:18:26,298 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:18:26,300 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:18:26,301 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:26,365 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:26,366 INFO MainThread:2647496 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:18:26,366 INFO MainThread:2647496 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:18:26,366 INFO MainThread:2647496 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:18:26,367 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:18:26,367 INFO HandlerThread:2647496 [handler.py:finish():882] shutting down handler +2025-01-21 21:18:27,298 INFO WriterThread:2647496 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/run-2o1us5j0.wandb +2025-01-21 21:18:27,366 INFO SenderThread:2647496 [sender.py:finish():1615] shutting down sender +2025-01-21 21:18:27,366 INFO SenderThread:2647496 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:18:27,366 INFO SenderThread:2647496 [file_pusher.py:join():175] waiting for file pusher diff --git a/2o1us5j0/wandb/debug.log b/2o1us5j0/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8cb21a71f5cba27ca6a3ec84b602c7b150587766 --- /dev/null +++ b/2o1us5j0/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Configure stats pid to 2647330 +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/settings +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug.log +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug-internal.log +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:13:39,115 INFO MainThread:2647330 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0'} +2025-01-21 21:13:39,115 INFO MainThread:2647330 [wandb_init.py:init():619] starting backend +2025-01-21 21:13:39,115 INFO MainThread:2647330 [wandb_init.py:init():623] setting up manager +2025-01-21 21:13:39,119 INFO MainThread:2647330 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:13:39,120 INFO MainThread:2647330 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:13:39,132 INFO MainThread:2647330 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:13:39,139 INFO MainThread:2647330 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:13:39,462 INFO MainThread:2647330 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:13:39,533 INFO MainThread:2647330 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:13:39,534 INFO MainThread:2647330 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:13:39,898 INFO MainThread:2647330 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:13:39,898 INFO MainThread:2647330 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:13:39,898 INFO MainThread:2647330 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:13:39,898 INFO MainThread:2647330 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:13:39,900 INFO MainThread:2647330 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:13:39,900 INFO MainThread:2647330 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/tb, True +2025-01-21 21:13:53,109 INFO MainThread:2647330 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:18:27,367 WARNING MsgRouterThr:2647330 [router.py:message_loop():77] message_loop has been closed diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code/FISH/eval_robot.py b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfa006801a5727ac412a1ab761d06345bdc631b0 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3465_0' +wandb_notes: + desc: null + value: 3465_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737512019 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/diff.patch b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..8da10724284d70d8fca0dedf572670b3116372fc --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/diff.patch @@ -0,0 +1,28 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..e77ad21 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -139,12 +139,12 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 06/20 demos per book + # crp_D+mask+act history +-# wandb_run_id: '3465_0' ++wandb_run_id: '3465_0' + # wandb_run_id: '3465_1' # seed 1 + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' ++# wandb_run_id: '4295_0' + # wandb_run_id: '4295_1' # seed 1 + # wandb_run_id: '4295_2' # seed 2 + diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json new file mode 100644 index 0000000000000000000000000000000000000000..3e0311d25d8258c6a2b001870ce1d0db46850295 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[57, 60, 0.95, 0.8482979088325087, 0.99116992195558, 0.8748771622027232, 0.9893508235876567, 0.888965532105854, 0.9814683794012344, 0.8955423998786676, 0.9806436010606424]]} \ No newline at end of file diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..52650538389a648a9da140f02e7bc0a9fc82408f --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b23de2781f443affd89f3f16a927cac8f5165c1720fe36a6bb9c646d5d4dbdb +size 1174690 diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d724de7b4b49a1dc7aa79970a801ddb0322e3d5a --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27ab141d29c0759686c9c2f7d491f49f02a175914b83f78068ff3dd45e2c9bbb +size 1174970 diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92e8de94c4dc9dafa122c80b75bdd89db7142e08 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02193a80ccc515d7b53d93235e42189751e8721c817731b26f7a96a753a9879d +size 1185158 diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2447900cfd631d470171cd4e7e4ac81f1cbe1730 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1146ecf321ca00c51feeeb4e45480b601f9ffa7208012261f29f436620d8a4d +size 1214627 diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..42b1d00283e87fcee0fab67ec3bc4357c0f2ab79 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c129977338f91caa371f496f88f6b0d2c4079eb1b71d2b5df643d085a7e841 +size 1195894 diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8ef73b85b92d706c78a0f01746fd10470b70f889 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log @@ -0,0 +1,269 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737512033.110554]: resetting environment +[INFO] [1737512033.113997]: cleared current plan +[INFO] [1737512033.115028]: moving to home +[INFO] [1737512034.117288]: reached home +[INFO] [1737512034.118888]: reset action history +[INFO] [1737512036.826581]: environment reset +Starting episode 0 +[INFO] [1737512036.828620]: resetting environment +[INFO] [1737512036.833621]: cleared current plan +[INFO] [1737512036.834777]: moving to home +[INFO] [1737512037.837212]: reached home +[INFO] [1737512037.838914]: reset action history +[INFO] [1737512040.547055]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737512051.317839]: resetting environment +[INFO] [1737512051.322533]: cleared current plan +[INFO] [1737512051.323405]: moving to home +[INFO] [1737512052.325258]: reached home +[INFO] [1737512052.327020]: reset action history +[INFO] [1737512055.033983]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/episode_rosbags/episode_0_2025-01-21-21-14-15.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2647330.log +started roslaunch server http://158.130.50.37:38247/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2647608] +process[print_text-2]: started with pid [2647632] +process[rosbag_record-3]: started with pid [2647633] +started bagging! +[INFO] [1737512082.054680]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737512082.800467]: Stored episode 1. +[INFO] [1737512082.801943]: Saving video... +[INFO] [1737512083.006385]: Video saved! +Starting episode 1 +[INFO] [1737512083.026349]: resetting environment +[INFO] [1737512083.033767]: cleared current plan +[INFO] [1737512083.034043]: moving to home +[INFO] [1737512086.637258]: reached home +[INFO] [1737512086.637763]: reset action history +[INFO] [1737512089.345840]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512101.193393]: resetting environment +[INFO] [1737512101.195740]: cleared current plan +[INFO] [1737512101.195926]: moving to home +[INFO] [1737512102.197393]: reached home +[INFO] [1737512102.197916]: reset action history +[INFO] [1737512104.904283]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/episode_rosbags/episode_1_2025-01-21-21-15-04.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2647330.log +started roslaunch server http://158.130.50.37:34789/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2647747] +process[print_text-5]: started with pid [2647748] +process[rosbag_record-6]: started with pid [2647772] +started bagging! +For topic gripper_width: timestamp difference is 113492089 for nearest: 1737512131712497550 - target: 1737512131599005461 at idx 249 +[INFO] [1737512131.883751]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737512132.626250]: Stored episode 2. +[INFO] [1737512132.626497]: Saving video... +[INFO] [1737512133.106348]: Video saved! +Starting episode 2 +[INFO] [1737512133.125534]: resetting environment +[INFO] [1737512133.132223]: cleared current plan +[INFO] [1737512133.138080]: moving to home +[INFO] [1737512136.745794]: reached home +[INFO] [1737512136.749430]: reset action history +[INFO] [1737512139.526770]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512143.314750]: resetting environment +[INFO] [1737512143.322287]: cleared current plan +[INFO] [1737512143.322573]: moving to home +[INFO] [1737512144.324523]: reached home +[INFO] [1737512144.324957]: reset action history +[INFO] [1737512147.032402]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/episode_rosbags/episode_2_2025-01-21-21-15-47.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2647330.log +started roslaunch server http://158.130.50.37:41111/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2647895] +process[print_text-8]: started with pid [2647896] +process[rosbag_record-9]: started with pid [2647897] +started bagging! +For topic gripper_width: timestamp difference is 102022763 for nearest: 1737512161579163190 - target: 1737512161477140427 at idx 201 +For topic gripper_width: timestamp difference is 110764396 for nearest: 1737512165979167831 - target: 1737512165868403435 at idx 249 +[INFO] [1737512174.169350]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737512175.003390]: Stored episode 3. +[INFO] [1737512175.010807]: Saving video... +[INFO] [1737512175.280802]: Video saved! +Starting episode 3 +[INFO] [1737512175.336191]: resetting environment +[INFO] [1737512175.369285]: cleared current plan +[INFO] [1737512175.372890]: moving to home +[INFO] [1737512178.978209]: reached home +[INFO] [1737512178.987720]: reset action history +[INFO] [1737512181.765712]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512184.808330]: resetting environment +[INFO] [1737512184.818248]: cleared current plan +[INFO] [1737512184.821200]: moving to home +[INFO] [1737512185.825378]: reached home +[INFO] [1737512185.826337]: reset action history +[INFO] [1737512188.609905]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/episode_rosbags/episode_3_2025-01-21-21-16-28.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2647330.log +started roslaunch server http://158.130.50.37:41483/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2648051] +process[print_text-11]: started with pid [2648052] +process[rosbag_record-12]: started with pid [2648053] +started bagging! +[INFO] [1737512216.822376]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1737512217.589358]: Stored episode 4. +[INFO] [1737512217.594275]: Saving video... +[INFO] [1737512218.007750]: Video saved! +Starting episode 4 +[INFO] [1737512218.057733]: resetting environment +[INFO] [1737512218.075688]: cleared current plan +[INFO] [1737512218.079482]: moving to home +[INFO] [1737512221.681863]: reached home +[INFO] [1737512221.684580]: reset action history +[INFO] [1737512224.466274]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512234.196702]: resetting environment +[INFO] [1737512234.223496]: cleared current plan +[INFO] [1737512234.229446]: moving to home +[INFO] [1737512235.237252]: reached home +[INFO] [1737512235.248843]: reset action history +[INFO] [1737512237.969909]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/episode_rosbags/episode_4_2025-01-21-21-17-17.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2647330.log +started roslaunch server http://158.130.50.37:41211/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2648181] +process[print_text-14]: started with pid [2648182] +process[rosbag_record-15]: started with pid [2648206] +started bagging! +For topic gripper_width: timestamp difference is 106640949 for nearest: 1737512248179162874 - target: 1737512248072521925 at idx 129 +For topic gripper_width: timestamp difference is 114048928 for nearest: 1737512254779164523 - target: 1737512254665115595 at idx 223 +For topic gripper_width: timestamp difference is 117134755 for nearest: 1737512259979162877 - target: 1737512259862028122 at idx 249 +[INFO] [1737512264.942516]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737512265.738468]: Stored episode 5. +[INFO] [1737512265.743216]: Saving video... +[INFO] [1737512266.268509]: Video saved! +[INFO] [1737512266.333812]: resetting environment +[INFO] [1737512266.350100]: cleared current plan +[INFO] [1737512266.350394]: moving to home +[WARN] [1737512266.317165]: Plan exhausted +[INFO] [1737512269.953169]: reached home +[INFO] [1737512269.960403]: reset action history +[INFO] [1737512272.806027]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/requirements.txt b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-metadata.json b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5848060b22820614c59f64a34aad84f78f9944ca --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:13:39.560261", + "startedAt": "2025-01-22T02:13:39.112325", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.98925, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4491.036, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 350.6014938354492 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7c64d9e56e7e9d909bc21df3b08e76dc858f6444 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "7b23de2781f443affd89f3f16a927cac8f5165c1720fe36a6bb9c646d5d4dbdb", "size": 1174690, "path": "media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4"}, "global_step": 665, "_timestamp": 1737512278.8579578, "_runtime": 259.7372329235077, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "27ab141d29c0759686c9c2f7d491f49f02a175914b83f78068ff3dd45e2c9bbb", "size": 1174970, "path": "media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4"}, "eval/num_success": 5.0, "episode": 60.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "02193a80ccc515d7b53d93235e42189751e8721c817731b26f7a96a753a9879d", "size": 1185158, "path": "media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d1146ecf321ca00c51feeeb4e45480b601f9ffa7208012261f29f436620d8a4d", "size": 1214627, "path": "media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "99c129977338f91caa371f496f88f6b0d2c4079eb1b71d2b5df643d085a7e841", "size": 1195894, "path": "media/videos/eval/4_eval_4_99c129977338f91caa37.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://kpgis3hvnsnccqtsmliwxdazqnq1qt1n80ixx4geucxm9qdkbk6oe09grlf2or2s3cpeh140amrn1ns3bm09qy30q20vrl6cteh771q59m9ycgmysrn2hdka5bkgolad/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://2xk392exoodgshxe0fobuxftnzaqbzfbjf4vkbmyyw8u1wnzcija3hc7m7cy07wuakk5u48o0wwg122s4cryzm8av19eew3zrk2f5i88hka36yhpi6gmml6avmdb3m54:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 57, "total_num_episodes": 60, "total_success_rate": 0.95, "total_success_rate_lb_0.01": 0.8482979088325087, "total_success_rate_ub_0.01": 0.99116992195558, "total_success_rate_lb_0.025": 0.8748771622027232, "total_success_rate_ub_0.025": 0.9893508235876567, "total_success_rate_lb_0.05": 0.888965532105854, "total_success_rate_ub_0.05": 0.9814683794012344, "total_success_rate_lb_0.1": 0.8955423998786676, "total_success_rate_ub_0.1": 0.9806436010606424, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "8e64a9a966331685f3584d18f79ab9def331cd7f7fdcae8c54d48aec961639b9", "size": 503, "artifact_path": "wandb-client-artifact://a4b7kivz90bge40yni6f3pd7i2qdfinzszby1mo7okfdvyncane4t3ch4p3mr57j6osxi8div88wmr7xsnh28lo9wqwcs1d824w29vgfghmqpa7t8k5fxdjiktwztbpw/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qll88p00khhf3ndjxcr9vllwqssqc4lisgv1oxnhg9rm8zon2wuo5yniv6kif92xl6vjb9pjs87x21x8xrrmxhabivg5pvqj47yzqkxhfkjnhguijeyh2o6zk5j5j6qs:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 277}, "eval/total_success_rate": 0.949999988079071, "eval/total_success_rate_lb_0.01": 0.8482978940010071, "eval/total_success_rate_ub_0.01": 0.9911699295043945, "eval/total_success_rate_lb_0.025": 0.8748771548271179, "eval/total_success_rate_ub_0.025": 0.9893507957458496, "eval/total_success_rate_lb_0.05": 0.8889655470848083, "eval/total_success_rate_ub_0.05": 0.9814683794975281, "eval/total_success_rate_lb_0.1": 0.8955423831939697, "eval/total_success_rate_ub_0.1": 0.980643630027771} \ No newline at end of file diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug-internal.log b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5da1ba763ce361a0dc9983a0cec92f81b6eb5b3c --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug-internal.log @@ -0,0 +1,785 @@ +2025-01-21 21:13:39,120 INFO StreamThr :2647496 [internal.py:wandb_internal():85] W&B internal server running at pid: 2647496, started at: 2025-01-21 21:13:39.120468 +2025-01-21 21:13:39,123 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:13:39,133 INFO WriterThread:2647496 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/run-2o1us5j0.wandb +2025-01-21 21:13:39,133 DEBUG SenderThread:2647496 [sender.py:send():379] send: header +2025-01-21 21:13:39,141 DEBUG SenderThread:2647496 [sender.py:send():379] send: run +2025-01-21 21:13:39,450 INFO SenderThread:2647496 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files +2025-01-21 21:13:39,450 INFO SenderThread:2647496 [sender.py:_start_run_threads():1188] run started: 2o1us5j0 with start time 1737512019.120725 +2025-01-21 21:13:39,462 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:13:39,462 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:13:39,541 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:13:39,546 DEBUG HandlerThread:2647496 [system_info.py:__init__():26] System info init +2025-01-21 21:13:39,546 DEBUG HandlerThread:2647496 [system_info.py:__init__():41] System info init done +2025-01-21 21:13:39,546 INFO HandlerThread:2647496 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:13:39,546 INFO SystemMonitor:2647496 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:13:39,547 INFO HandlerThread:2647496 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:13:39,547 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:13:39,548 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:13:39,549 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:13:39,549 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:13:39,549 INFO SystemMonitor:2647496 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:13:39,560 DEBUG HandlerThread:2647496 [system_info.py:probe():152] Probing system +2025-01-21 21:13:39,561 DEBUG HandlerThread:2647496 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:13:39,566 DEBUG HandlerThread:2647496 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:13:39,566 DEBUG HandlerThread:2647496 [system_info.py:probe():200] Probing system done +2025-01-21 21:13:39,566 DEBUG HandlerThread:2647496 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:13:39.560261', 'startedAt': '2025-01-22T02:13:39.112325', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.98925, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4491.036, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 350.6014938354492}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:13:39,566 INFO HandlerThread:2647496 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:13:39,566 INFO HandlerThread:2647496 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:13:39,566 DEBUG HandlerThread:2647496 [system_info.py:_save_code():44] Saving code +2025-01-21 21:13:39,567 DEBUG HandlerThread:2647496 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:13:39,567 DEBUG HandlerThread:2647496 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:13:39,629 DEBUG HandlerThread:2647496 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:13:39,630 INFO HandlerThread:2647496 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:13:39,633 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:13:39,633 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:13:39,633 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:13:39,633 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:13:39,897 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:13:39,897 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:13:39,899 DEBUG SenderThread:2647496 [sender.py:send():379] send: telemetry +2025-01-21 21:13:39,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:13:39,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:39,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:13:39,902 INFO HandlerThread:2647496 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/tb" + save: true +} +control { + relay_id: "07530a4f366b4424bd90a669e2ceac72" +} +_info { + stream_id: "2o1us5j0" +} + +2025-01-21 21:13:39,902 DEBUG HandlerThread:2647496 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:13:39,926 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/fxfagu85-diff.patch +2025-01-21 21:13:39,928 INFO wandb-upload_1:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/yvoo1sr9-code/FISH/eval_robot.py +2025-01-21 21:13:39,938 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/rtiscy5w-wandb-metadata.json +2025-01-21 21:13:40,029 DEBUG SenderThread:2647496 [sender.py:send():379] send: tbrecord +2025-01-21 21:13:40,029 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:13:40,029 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 with policy live +2025-01-21 21:13:40,212 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/6be263eg-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/diff.patch +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-metadata.json +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/requirements.txt +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code/FISH/eval_robot.py +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code +2025-01-21 21:13:40,452 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code/FISH +2025-01-21 21:13:40,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:41,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:42,451 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:42,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:43,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:44,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:13:44,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:45,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:46,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:47,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:48,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:49,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:13:49,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:50,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:51,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:52,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:53,110 DEBUG SenderThread:2647496 [sender.py:send():379] send: config +2025-01-21 21:13:53,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:54,454 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:54,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:13:54,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:13:54,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:54,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:13:55,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:56,455 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:56,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:57,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:58,455 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:13:58,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:13:59,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:00,456 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:00,547 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:00,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:01,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:02,457 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:02,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:03,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:04,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:05,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:05,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:06,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:07,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:08,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:09,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:14:09,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:14:09,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:10,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:11,004 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:11,459 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml +2025-01-21 21:14:11,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:12,459 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:12,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:13,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:14,460 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:14,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:15,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:16,327 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:16,461 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:16,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:17,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:18,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:19,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:20,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:21,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:21,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:22,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:23,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:24,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:14:24,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:14:24,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:25,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:26,904 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:26,988 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:27,905 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:28,905 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:29,905 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:30,905 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:31,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:32,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:32,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:33,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:34,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:35,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:36,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:37,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:38,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:38,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:39,550 DEBUG SystemMonitor:2647496 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:14:39,550 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:14:39,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:14:39,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:14:39,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:40,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:41,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:42,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:43,049 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:14:43,050 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:14:43,050 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 with policy now +2025-01-21 21:14:43,050 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:14:43,051 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:14:43,051 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:14:43,299 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/nhde7sj8-media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 +2025-01-21 21:14:43,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:14:43,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 +2025-01-21 21:14:43,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos +2025-01-21 21:14:43,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:14:43,470 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media +2025-01-21 21:14:43,767 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/rhq817hg-media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 +2025-01-21 21:14:43,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:44,051 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:44,469 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:44,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:45,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:46,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:47,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:48,471 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:48,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:49,346 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:49,908 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:50,471 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:14:50,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:51,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:52,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:53,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:54,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:14:54,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:14:54,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:55,033 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:14:55,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:56,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:57,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:58,909 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:14:59,911 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:00,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:00,910 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:01,475 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:15:01,733 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/77lcrimn-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:15:01,910 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:02,475 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:02,911 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:03,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:04,476 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:04,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:05,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:06,156 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:06,477 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:06,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:07,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:08,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:09,551 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:15:09,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:15:09,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:15:09,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:10,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:11,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:12,035 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:12,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:13,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:14,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:15,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:16,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:17,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:17,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:18,913 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:19,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:20,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:21,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:22,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:22,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:23,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:24,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:15:24,899 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:15:24,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:25,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:26,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:27,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:27,990 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:28,914 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:29,915 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:30,915 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:31,915 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:32,915 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:33,107 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:33,153 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:15:33,154 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:15:33,154 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 with policy now +2025-01-21 21:15:33,154 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:15:33,154 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:15:33,157 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:15:33,395 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/16b80rlp-media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 +2025-01-21 21:15:33,485 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:15:33,486 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 +2025-01-21 21:15:33,486 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:15:33,785 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/z7rklwx6-media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 +2025-01-21 21:15:33,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:34,486 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:34,919 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:35,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:36,919 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:37,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:38,487 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:38,789 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:38,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:39,551 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:15:39,898 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:15:39,898 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:15:39,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:40,488 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:40,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:41,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:42,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:43,489 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:15:43,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:44,333 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:44,339 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:15:44,339 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:15:44,339 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:15:44,489 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:15:44,490 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:44,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:45,918 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:46,490 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:46,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:47,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:48,491 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:15:48,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:49,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:49,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:50,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:51,920 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:52,921 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:53,921 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:54,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:15:54,899 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:15:54,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:54,973 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:15:55,927 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:56,927 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:57,927 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:58,927 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:15:59,928 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:00,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:00,929 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:01,929 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:02,496 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:02,930 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:03,930 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:04,930 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:05,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:05,930 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:06,931 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:07,931 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:08,497 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:08,931 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:09,552 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:16:09,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:16:09,899 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:16:09,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:10,936 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:11,011 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:11,937 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:12,937 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:13,937 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:14,946 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:15,329 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:16:15,329 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 with policy now +2025-01-21 21:16:15,334 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:16:15,334 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:16:15,334 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:16:15,334 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:16:15,500 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:16:15,500 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 +2025-01-21 21:16:15,500 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:16:15,566 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/bhs56rj6-media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 +2025-01-21 21:16:15,839 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/xtywq2ny-media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 +2025-01-21 21:16:15,946 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:16,373 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:16,500 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:16,946 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:17,942 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:18,942 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:19,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:20,501 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:20,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:21,767 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:21,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:22,945 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:23,948 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:24,503 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:24,902 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:16:24,902 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:16:24,953 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:25,156 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:16:25,156 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:16:25,156 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:16:25,503 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:16:25,503 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:16:25,767 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/eblg8jeh-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:16:25,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:26,503 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:26,869 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:26,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:27,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:28,504 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:28,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:29,948 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:30,505 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:30,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:31,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:32,107 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:32,506 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:32,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:33,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:34,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:35,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:36,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:37,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:37,949 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:38,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:39,552 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:16:39,899 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:16:39,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:16:39,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:40,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:41,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:42,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:42,981 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:43,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:44,950 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:45,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:46,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:47,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:48,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:48,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:49,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:50,951 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:51,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:52,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:53,907 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:53,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:54,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:16:54,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:16:54,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:55,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:56,953 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:57,954 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:58,050 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:16:58,050 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 with policy now +2025-01-21 21:16:58,053 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:16:58,053 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:16:58,053 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:16:58,053 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:16:58,299 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/vpsz4ygw-media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 +2025-01-21 21:16:58,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:16:58,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:16:58,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 +2025-01-21 21:16:58,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:16:58,801 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/eqn0vb4m-media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 +2025-01-21 21:16:58,955 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:16:59,080 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:16:59,954 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:00,515 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:00,958 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:01,966 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:02,516 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:02,962 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:03,960 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:04,470 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:04,954 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:05,956 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:06,517 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:06,960 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:07,964 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:08,966 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:09,553 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:17:09,553 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:09,906 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:17:09,907 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:17:09,967 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:10,965 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:11,973 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:12,973 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:13,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:14,211 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:17:14,211 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:14,212 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:14,520 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:17:14,520 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:17:14,912 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:14,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:15,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:16,521 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:16,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:18,017 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:18,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:19,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:20,522 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:20,911 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:20,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:21,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:22,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:23,968 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:24,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:17:24,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:17:24,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:25,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:26,061 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:26,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:27,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:28,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:29,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:30,525 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:30,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:31,911 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:31,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:32,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:33,969 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:34,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:35,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:36,527 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:36,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:36,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:37,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:38,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:39,553 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:17:39,900 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:17:39,900 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:17:39,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:40,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:41,970 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:42,041 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:42,530 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:42,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:43,971 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:44,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:45,975 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:46,326 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:17:46,327 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 with policy now +2025-01-21 21:17:46,333 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:17:46,333 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:17:46,333 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:46,334 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:46,531 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:17:46,531 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:46,531 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 +2025-01-21 21:17:46,531 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval +2025-01-21 21:17:46,555 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/i1jwc2vc-media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 +2025-01-21 21:17:46,824 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/mrdlrkek-media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 +2025-01-21 21:17:46,978 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:47,350 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:47,976 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:48,532 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:48,978 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:50,005 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:50,983 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:51,982 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:52,533 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:52,807 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:52,982 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:53,980 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:54,534 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:54,926 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:17:54,926 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:17:54,994 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:55,992 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:56,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:57,535 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:17:57,798 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/c9psj602-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:17:57,995 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:58,267 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:17:58,267 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:58,267 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:17:58,268 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:58,342 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:17:58,342 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:17:58,535 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table +2025-01-21 21:17:58,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval +2025-01-21 21:17:58,549 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:17:58,730 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/3cpmjebx-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:17:58,825 INFO wandb-upload_0:2647496 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3ejflfr6 +2025-01-21 21:17:58,941 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:17:58,952 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:17:58,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:17:59,232 INFO SenderThread:2647496 [sender.py:send_request_log_artifact():1525] logged artifact run-2o1us5j0-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY1ODIxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5Mzc4MA==', 'latestArtifact': None}} +2025-01-21 21:17:59,233 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:17:59,233 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-21 21:17:59,233 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:17:59,233 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,234 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,234 DEBUG SenderThread:2647496 [sender.py:send():379] send: telemetry +2025-01-21 21:17:59,234 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,234 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,234 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,235 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,235 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,237 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,237 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,237 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,237 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,238 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,238 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,238 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,238 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,239 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,239 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,239 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,239 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,240 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,240 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,240 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,240 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:17:59,240 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:17:59,241 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:17:59,495 INFO wandb-upload_1:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/mvi0bfk2-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:17:59,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:17:59,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:17:59,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json +2025-01-21 21:17:59,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval +2025-01-21 21:17:59,587 INFO wandb-upload_0:2647496 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjo3psi_s +2025-01-21 21:17:59,718 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/u5jhwjet-media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json +2025-01-21 21:17:59,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:00,109 INFO SenderThread:2647496 [sender.py:send_request_log_artifact():1525] logged artifact run-2o1us5j0-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY1ODIyOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5Mzc4Ng==', 'latestArtifact': None}} +2025-01-21 21:18:00,109 DEBUG SenderThread:2647496 [sender.py:send():379] send: files +2025-01-21 21:18:00,109 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json with policy now +2025-01-21 21:18:00,109 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:18:00,109 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:18:00,110 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:18:00,366 INFO wandb-upload_1:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/hkyl84mn-media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json +2025-01-21 21:18:00,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:18:00,536 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:18:00,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:01,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:02,984 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:03,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:04,111 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:04,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:05,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:06,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:07,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:08,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:09,554 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:18:09,555 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:09,901 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:18:09,901 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:18:09,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:10,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:11,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:12,985 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:13,986 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:14,917 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:14,986 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:15,986 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:16,901 DEBUG SenderThread:2647496 [sender.py:send():379] send: exit +2025-01-21 21:18:16,902 INFO SenderThread:2647496 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:18:16,902 INFO SenderThread:2647496 [sender.py:send_exit():588] handling runtime: 277 +2025-01-21 21:18:16,902 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:send_exit():594] send defer +2025-01-21 21:18:16,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,903 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:18:16,903 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:18:16,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,903 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:18:16,903 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:18:16,903 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:18:16,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,904 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:18:16,904 INFO HandlerThread:2647496 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:18:16,904 DEBUG SystemMonitor:2647496 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:18:16,904 DEBUG SystemMonitor:2647496 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:18:16,904 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:18:16,905 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:18:16,957 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:18:16,957 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:18:16,958 INFO HandlerThread:2647496 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:18:16,958 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:16,958 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:18:16,958 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:18:16,958 DEBUG SenderThread:2647496 [sender.py:send():379] send: stats +2025-01-21 21:18:16,959 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,959 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:18:16,959 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:16,959 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:18:16,959 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:18:16,960 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:16,960 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:18:17,543 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:18:18,544 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:18:23,274 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:23,274 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:23,274 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:18:23,276 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:23,276 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:18:23,276 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:18:23,276 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:23,276 DEBUG SenderThread:2647496 [sender.py:send():379] send: history +2025-01-21 21:18:23,276 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:18:23,276 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:23,277 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:18:23,278 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:18:23,278 DEBUG SenderThread:2647496 [sender.py:send():379] send: summary +2025-01-21 21:18:23,282 INFO SenderThread:2647496 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:18:23,282 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:23,282 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:18:23,282 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:18:23,282 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:23,282 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:18:23,282 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:23,282 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:18:23,286 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:23,549 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:18:23,549 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:23,550 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:18:23,550 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:23,550 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:18:23,550 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:18:23,550 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml +2025-01-21 21:18:23,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:24,357 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:18:24,357 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:24,357 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:24,358 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:18:24,358 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:24,358 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:18:24,358 INFO SenderThread:2647496 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:18:24,360 INFO SenderThread:2647496 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:18:24,363 INFO SenderThread:2647496 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:18:24,373 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:18:24,373 DEBUG SenderThread:2647496 [sender.py:send():379] send: artifact +2025-01-21 21:18:24,373 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:24,376 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:18:24,551 INFO Thread-12 :2647496 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:18:24,565 INFO SenderThread:2647496 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:18:24,565 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:24,565 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:18:24,565 INFO SenderThread:2647496 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:18:24,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml config.yaml +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json wandb-summary.json +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/requirements.txt requirements.txt +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:18:25,552 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log output.log +2025-01-21 21:18:25,557 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/diff.patch diff.patch +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 media/videos/eval/3_eval_3_d1146ecf321ca00c51fe.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 media/videos/eval/0_eval_0_7b23de2781f443affd89.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 media/videos/eval/1_eval_1_27ab141d29c0759686c9.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 media/videos/eval/2_eval_2_02193a80ccc515d7b53d.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 media/videos/eval/4_eval_4_99c129977338f91caa37.mp4 +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json media/table/eval/total_success_rate_ci_6_8e64a9a966331685f358.table.json +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:18:25,558 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:18:25,559 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:25,559 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:25,559 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:18:25,559 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:25,559 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:18:25,559 INFO SenderThread:2647496 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:18:25,746 INFO wandb-upload_2:2647496 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/config.yaml +2025-01-21 21:18:25,758 INFO wandb-upload_1:2647496 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/requirements.txt +2025-01-21 21:18:25,774 INFO wandb-upload_4:2647496 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/output.log +2025-01-21 21:18:25,775 INFO wandb-upload_0:2647496 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/files/wandb-summary.json +2025-01-21 21:18:25,884 INFO wandb-upload_3:2647496 [upload_job.py:push():130] Uploaded file /tmp/tmpi7mqcor2wandb/k7rh9v1y-events.out.tfevents.1737512019.leonmkim-ROG-Strix-G15CS-G15CS.2647330.0 +2025-01-21 21:18:25,903 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:25,903 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:26,085 INFO Thread-11 (_thread_body):2647496 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:18:26,085 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:26,085 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:18:26,085 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:26,085 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:18:26,085 INFO SenderThread:2647496 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:18:26,087 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:18:26,087 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:26,087 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:18:26,087 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:26,087 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:18:26,087 INFO SenderThread:2647496 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:18:26,295 INFO SenderThread:2647496 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:18:26,295 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:18:26,296 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:26,296 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:18:26,296 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:26,296 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:18:26,296 INFO SenderThread:2647496 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:18:26,296 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:18:26,296 INFO HandlerThread:2647496 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:18:26,296 DEBUG SenderThread:2647496 [sender.py:send():379] send: final +2025-01-21 21:18:26,296 DEBUG SenderThread:2647496 [sender.py:send():379] send: footer +2025-01-21 21:18:26,297 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: defer +2025-01-21 21:18:26,297 INFO SenderThread:2647496 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:18:26,297 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:26,297 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:18:26,297 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:26,298 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:18:26,298 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:18:26,298 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:18:26,300 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:18:26,301 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:26,365 DEBUG SenderThread:2647496 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:18:26,366 INFO MainThread:2647496 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:18:26,366 INFO MainThread:2647496 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:18:26,366 INFO MainThread:2647496 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:18:26,367 DEBUG HandlerThread:2647496 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:18:26,367 INFO HandlerThread:2647496 [handler.py:finish():882] shutting down handler +2025-01-21 21:18:27,298 INFO WriterThread:2647496 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/run-2o1us5j0.wandb +2025-01-21 21:18:27,366 INFO SenderThread:2647496 [sender.py:finish():1615] shutting down sender +2025-01-21 21:18:27,366 INFO SenderThread:2647496 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:18:27,366 INFO SenderThread:2647496 [file_pusher.py:join():175] waiting for file pusher diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug.log b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8cb21a71f5cba27ca6a3ec84b602c7b150587766 --- /dev/null +++ b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Configure stats pid to 2647330 +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/settings +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug.log +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/logs/debug-internal.log +2025-01-21 21:13:39,114 INFO MainThread:2647330 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:13:39,115 INFO MainThread:2647330 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0'} +2025-01-21 21:13:39,115 INFO MainThread:2647330 [wandb_init.py:init():619] starting backend +2025-01-21 21:13:39,115 INFO MainThread:2647330 [wandb_init.py:init():623] setting up manager +2025-01-21 21:13:39,119 INFO MainThread:2647330 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:13:39,120 INFO MainThread:2647330 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:13:39,132 INFO MainThread:2647330 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:13:39,139 INFO MainThread:2647330 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:13:39,462 INFO MainThread:2647330 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:13:39,533 INFO MainThread:2647330 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:13:39,534 INFO MainThread:2647330 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:13:39,898 INFO MainThread:2647330 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:13:39,898 INFO MainThread:2647330 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:13:39,898 INFO MainThread:2647330 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:13:39,898 INFO MainThread:2647330 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:13:39,900 INFO MainThread:2647330 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:13:39,900 INFO MainThread:2647330 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/2o1us5j0/tb, True +2025-01-21 21:13:53,109 INFO MainThread:2647330 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:18:27,367 WARNING MsgRouterThr:2647330 [router.py:message_loop():77] message_loop has been closed diff --git a/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/run-2o1us5j0.wandb b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/run-2o1us5j0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f3c0d7e9a2e955b61be15139cd16eca474c394d6 Binary files /dev/null and b/2o1us5j0/wandb/run-20250121_211339-2o1us5j0/run-2o1us5j0.wandb differ diff --git a/mo0b6mh9/.hydra/config.yaml b/mo0b6mh9/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1f05e6fe3d0ebd009320b13cbc68f24b3ac1661 --- /dev/null +++ b/mo0b6mh9/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/mo0b6mh9/.hydra/hydra.yaml b/mo0b6mh9/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f60c56219b2eee6560e1f936a989e2254300fcf --- /dev/null +++ b/mo0b6mh9/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/mo0b6mh9/.hydra/overrides.yaml b/mo0b6mh9/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/mo0b6mh9/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/mo0b6mh9/episode_rosbags/color_K.npy b/mo0b6mh9/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/mo0b6mh9/episode_rosbags/episode_0_2025-01-18-19-55-45.bag b/mo0b6mh9/episode_rosbags/episode_0_2025-01-18-19-55-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..fcc0c473549b2ded76f1f8d2134264d3ee04ac75 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_0_2025-01-18-19-55-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adb44f23cd713aab7a4d7d7ff219d1cb135bb4e28b24d3460095757913c90b3b +size 1408010139 diff --git a/mo0b6mh9/episode_rosbags/episode_11_2025-01-18-20-03-53.bag b/mo0b6mh9/episode_rosbags/episode_11_2025-01-18-20-03-53.bag new file mode 100644 index 0000000000000000000000000000000000000000..179c7acd6e35006406869330dff17de876e218f4 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_11_2025-01-18-20-03-53.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09e57b7bf024b94f65bf6e3fdfe928b0afae8eaa4ec8d317f1414dbc6dae591d +size 1407756886 diff --git a/mo0b6mh9/episode_rosbags/episode_12_2025-01-18-20-04-42.bag b/mo0b6mh9/episode_rosbags/episode_12_2025-01-18-20-04-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..e630a8dce2c45f37e46c30e38d4b3949784c7f30 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_12_2025-01-18-20-04-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7f68497d5009978c093b4e029c5b510265796b7d3cfcf612d598c21b42eac11 +size 1413770355 diff --git a/mo0b6mh9/episode_rosbags/episode_16_2025-01-18-20-08-36.bag b/mo0b6mh9/episode_rosbags/episode_16_2025-01-18-20-08-36.bag new file mode 100644 index 0000000000000000000000000000000000000000..100a9632f93eb5582e8376e6f171ad605b723d05 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_16_2025-01-18-20-08-36.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37a068939899f667680221db5184ac8232f0752d9724dfa49b57ed4939029a86 +size 1415292322 diff --git a/mo0b6mh9/episode_rosbags/episode_17_2025-01-18-20-09-31.bag b/mo0b6mh9/episode_rosbags/episode_17_2025-01-18-20-09-31.bag new file mode 100644 index 0000000000000000000000000000000000000000..d12bb42783411ccaa7763994feea6bdf2651362c --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_17_2025-01-18-20-09-31.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15674b5c3099047e0ae0ba7fb3a67b077d9a7f29d11c50840ca4b11469fbf0df +size 1412741717 diff --git a/mo0b6mh9/episode_rosbags/episode_2_2025-01-18-19-57-07.bag b/mo0b6mh9/episode_rosbags/episode_2_2025-01-18-19-57-07.bag new file mode 100644 index 0000000000000000000000000000000000000000..189688d1b1225cef7dbd2ff5366356ceb7f3646b --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_2_2025-01-18-19-57-07.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72780fb9ac6116dce930e07250f6bacb35f8b327f5c49ae4ea17c158a2e76a95 +size 1412396526 diff --git a/mo0b6mh9/episode_rosbags/episode_4_2025-01-18-19-58-35.bag b/mo0b6mh9/episode_rosbags/episode_4_2025-01-18-19-58-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..0d9fbb7bff65cb68c5d5c864e1383e7d5de5afd8 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_4_2025-01-18-19-58-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efbe32864b4f891d1bb665286cf44382eebf17745bb7c96977b256bf0c4bbb25 +size 1413023757 diff --git a/mo0b6mh9/episode_rosbags/episode_5_2025-01-18-19-59-26.bag b/mo0b6mh9/episode_rosbags/episode_5_2025-01-18-19-59-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..70cd6f31c6d5165d57005ccc88116e71b7aa3992 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_5_2025-01-18-19-59-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22c1a4f8a5d9f99232af94607e3c390e1b009b5797beff61b02efb67836273de +size 1413436671 diff --git a/mo0b6mh9/episode_rosbags/episode_8_2025-01-18-20-01-38.bag b/mo0b6mh9/episode_rosbags/episode_8_2025-01-18-20-01-38.bag new file mode 100644 index 0000000000000000000000000000000000000000..f0d229030778ca3fcd41beb19ab4e0743ffcb21b --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_8_2025-01-18-20-01-38.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52a04fc23f72391039316d78204daac25b3b5b11f2a6a6c574533b5ecfae1c3a +size 1410746354 diff --git a/mo0b6mh9/episode_rosbags/episode_9_2025-01-18-20-02-21.bag b/mo0b6mh9/episode_rosbags/episode_9_2025-01-18-20-02-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..1981a75f197dc7216127f79f0168a02494954d23 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_9_2025-01-18-20-02-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bcd963275130a264092ba6c0c459fb17984e16b3740fd463d60ed2a02a737cc +size 1409898570 diff --git a/mo0b6mh9/eval_video/0_eval.mp4 b/mo0b6mh9/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7720b7930a725afa338b6988b7ac911aff3407aa --- /dev/null +++ b/mo0b6mh9/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd640c135955b3d54fec7ae6fb41c2edef3b17fda1044801e3101daccb7fcae9 +size 1185117 diff --git a/mo0b6mh9/eval_video/10_eval.mp4 b/mo0b6mh9/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da10d4e37e93db6ad5c26cea42a67152c526e7ee --- /dev/null +++ b/mo0b6mh9/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:708e5d86baaf5c876d21100f22ad7b92231cbe37549a12b676e35762ac00de0b +size 1369725 diff --git a/mo0b6mh9/eval_video/11_eval.mp4 b/mo0b6mh9/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e3e6216017c05cf3cd57ad8d7d04c09f2f8f07e7 --- /dev/null +++ b/mo0b6mh9/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e48fceb469367c5942f582757e64e07e7cccfbdf6b03d7970d1ccbe00037983 +size 1401639 diff --git a/mo0b6mh9/eval_video/12_eval.mp4 b/mo0b6mh9/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4273b35b4910d6e3b36f6b053c5355e8e5bc8506 --- /dev/null +++ b/mo0b6mh9/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a56ddf9bcd287e59382b84cd2c72971de6aaa276cbfe72b32f05001e3a2a734 +size 1170389 diff --git a/mo0b6mh9/eval_video/13_eval.mp4 b/mo0b6mh9/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2150d40d7f69f604f79712478a8c3b2eed4d2859 --- /dev/null +++ b/mo0b6mh9/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce205101458cc00950f123e1c7e065c5f0df37ab903d9ab26f3e942715d9d157 +size 1488237 diff --git a/mo0b6mh9/eval_video/14_eval.mp4 b/mo0b6mh9/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15cb9b16a42f7db710b5f755c0592255b292ac7c --- /dev/null +++ b/mo0b6mh9/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf5561b5a4f98495a05489e319bb0dd738c06da1ea4e5a26102d02c2dc6be54b +size 1480575 diff --git a/mo0b6mh9/eval_video/15_eval.mp4 b/mo0b6mh9/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6aa92990a709606ab94f998dd937306042117382 --- /dev/null +++ b/mo0b6mh9/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2535ee93601a7918b8bfce14005fc87a9d3bcebbb4b0971083f484b8e6bf71d8 +size 1535847 diff --git a/mo0b6mh9/eval_video/16_eval.mp4 b/mo0b6mh9/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..150f5f39cb658be85ed9bbd07f6feef507e504ae --- /dev/null +++ b/mo0b6mh9/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e356bbb63bbb79e9934d343610bc8c64258a145e769e19279f1394bf799a694 +size 1565153 diff --git a/mo0b6mh9/eval_video/17_eval.mp4 b/mo0b6mh9/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fbb469446466f94ab944d0d3d72911f9cde5b2ae --- /dev/null +++ b/mo0b6mh9/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21895a4eb18f20222f93e9c997afd5ed44a67fc6032cb002d203cb3f84fc63ac +size 1184885 diff --git a/mo0b6mh9/eval_video/18_eval.mp4 b/mo0b6mh9/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eeebb9398fc61fc87cc3179278995f3f961d1bf8 --- /dev/null +++ b/mo0b6mh9/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76a7bdc1f11678727d6f4fb49e1c74b004607ee6ee76205af6a182c5bfca6070 +size 1156589 diff --git a/mo0b6mh9/eval_video/19_eval.mp4 b/mo0b6mh9/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..750c25a7ef0faecb36dc3b87956ac7b0e8658184 --- /dev/null +++ b/mo0b6mh9/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dc41dd83150fd316589e669dd4ead5d4a0f7cefd969fc964a8154786b5403a0 +size 1164776 diff --git a/mo0b6mh9/eval_video/1_eval.mp4 b/mo0b6mh9/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4a30cffc34d96d8609b1badf911f991e09832b0c --- /dev/null +++ b/mo0b6mh9/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f01db884f36bf3d225fe9af9bfdda114326913abf015553d8f07a4b38a867f5d +size 1156684 diff --git a/mo0b6mh9/eval_video/2_eval.mp4 b/mo0b6mh9/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c777e9a73e99966964d11833d4db5eca8412d781 --- /dev/null +++ b/mo0b6mh9/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:593caa1fa9ea606eafa56a443e6ab8b0704d6bbc3a0866c02538c2b1214d14ef +size 1168318 diff --git a/mo0b6mh9/eval_video/3_eval.mp4 b/mo0b6mh9/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..358bdf3a5d4b580b1f90550144ac18b881596370 --- /dev/null +++ b/mo0b6mh9/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05651e72bfc1fe624e25ddb4c9d00aa112f0ff67b6a7bbe2a4e4ba3c0ad03806 +size 1167818 diff --git a/mo0b6mh9/eval_video/4_eval.mp4 b/mo0b6mh9/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0606b062fd78d1b249bcdf69df93f358bcbd4bf9 --- /dev/null +++ b/mo0b6mh9/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09e9e69382c41ac17586dafc07c5626df519b3bc99dd557ed1209c51ee2a146f +size 1192078 diff --git a/mo0b6mh9/eval_video/5_eval.mp4 b/mo0b6mh9/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a59bd85864b9e18331ab1404490272827430d9ca --- /dev/null +++ b/mo0b6mh9/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d80af6b216e8c15164e950c5bca51c718d88b8343cabba676337bc9f998a21f6 +size 1174056 diff --git a/mo0b6mh9/eval_video/6_eval.mp4 b/mo0b6mh9/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d5b1aba80f1e0798ebcdb7c94e99a94d420a2185 --- /dev/null +++ b/mo0b6mh9/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3602d914de1709b10240c32b6e0bb9ee74a8ec5bfda12121d3742a71c7d34246 +size 1182231 diff --git a/mo0b6mh9/eval_video/7_eval.mp4 b/mo0b6mh9/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..501646fccaa4fd2ef38994f4e010818412496c9b --- /dev/null +++ b/mo0b6mh9/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:960ad0adbe0863fa34f9b5075f6b3fb487f931c8f4793fb75e676e9ad15c51a0 +size 1313978 diff --git a/mo0b6mh9/eval_video/8_eval.mp4 b/mo0b6mh9/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ed1edff532c4589db55df23a786ed1d0e926672 --- /dev/null +++ b/mo0b6mh9/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a18bdd252b8f5e209e5b888cfe1c153b21be9d07924f54d1b6104d0beb21db7 +size 1160246 diff --git a/mo0b6mh9/eval_video/9_eval.mp4 b/mo0b6mh9/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..94f67b98bc59beb3496c6abe90e7ce46c5685f93 --- /dev/null +++ b/mo0b6mh9/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeb6222427bdaa35d11b90fdd007723e3f9b0fc60ce0c7e6a69608e1355f480f +size 1186279 diff --git a/mo0b6mh9/tb/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 b/mo0b6mh9/tb/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 new file mode 100644 index 0000000000000000000000000000000000000000..41ef73b87e81710bf0da29a521306b585cd45fbd --- /dev/null +++ b/mo0b6mh9/tb/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0964d92f3d63bb2dfcbe64a78419ed92b1647b74381b32ed224674a2661992d +size 22784 diff --git a/mo0b6mh9/wandb/debug-internal.log b/mo0b6mh9/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f0388f5cf579ca3fdd992ee4d39acb0128c6891d --- /dev/null +++ b/mo0b6mh9/wandb/debug-internal.log @@ -0,0 +1,2199 @@ +2025-01-18 19:54:57,688 INFO StreamThr :2566524 [internal.py:wandb_internal():85] W&B internal server running at pid: 2566524, started at: 2025-01-18 19:54:57.688322 +2025-01-18 19:54:57,689 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status +2025-01-18 19:54:57,701 INFO WriterThread:2566524 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/run-mo0b6mh9.wandb +2025-01-18 19:54:57,701 DEBUG SenderThread:2566524 [sender.py:send():379] send: header +2025-01-18 19:54:57,710 DEBUG SenderThread:2566524 [sender.py:send():379] send: run +2025-01-18 19:54:57,948 INFO SenderThread:2566524 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files +2025-01-18 19:54:57,949 INFO SenderThread:2566524 [sender.py:_start_run_threads():1188] run started: mo0b6mh9 with start time 1737248097.688871 +2025-01-18 19:54:57,963 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 19:54:57,963 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: check_version +2025-01-18 19:54:58,036 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 19:54:58,044 DEBUG HandlerThread:2566524 [system_info.py:__init__():26] System info init +2025-01-18 19:54:58,044 DEBUG HandlerThread:2566524 [system_info.py:__init__():41] System info init done +2025-01-18 19:54:58,045 INFO HandlerThread:2566524 [system_monitor.py:start():194] Starting system monitor +2025-01-18 19:54:58,045 INFO SystemMonitor:2566524 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 19:54:58,045 INFO HandlerThread:2566524 [system_monitor.py:probe():214] Collecting system info +2025-01-18 19:54:58,045 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 19:54:58,047 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started disk monitoring +2025-01-18 19:54:58,047 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 19:54:58,048 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started memory monitoring +2025-01-18 19:54:58,048 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started network monitoring +2025-01-18 19:54:58,058 DEBUG HandlerThread:2566524 [system_info.py:probe():152] Probing system +2025-01-18 19:54:58,060 DEBUG HandlerThread:2566524 [system_info.py:_probe_git():137] Probing git +2025-01-18 19:54:58,064 DEBUG HandlerThread:2566524 [system_info.py:_probe_git():145] Probing git done +2025-01-18 19:54:58,065 DEBUG HandlerThread:2566524 [system_info.py:probe():200] Probing system done +2025-01-18 19:54:58,065 DEBUG HandlerThread:2566524 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T00:54:58.059031', 'startedAt': '2025-01-19T00:54:57.673346', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3559.8128749999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 4489.241, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 579.0002593994141}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 19:54:58,065 INFO HandlerThread:2566524 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 19:54:58,065 INFO HandlerThread:2566524 [system_monitor.py:probe():227] Publishing system info +2025-01-18 19:54:58,065 DEBUG HandlerThread:2566524 [system_info.py:_save_code():44] Saving code +2025-01-18 19:54:58,070 DEBUG HandlerThread:2566524 [system_info.py:_save_code():67] Saving code done +2025-01-18 19:54:58,070 DEBUG HandlerThread:2566524 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 19:54:58,168 DEBUG HandlerThread:2566524 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 19:54:58,169 INFO HandlerThread:2566524 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 19:54:58,172 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:54:58,172 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 19:54:58,172 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 19:54:58,173 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 19:54:58,173 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 19:54:58,402 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 19:54:58,403 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: python_packages +2025-01-18 19:54:58,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:54:58,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:54:58,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:54:58,435 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/vx0a3pt4-diff.patch +2025-01-18 19:54:58,438 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ibb2j8co-wandb-metadata.json +2025-01-18 19:54:58,445 INFO HandlerThread:2566524 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/tb" + save: true +} +control { + relay_id: "70c5eacc984a4afebe7199c0bf475f13" +} +_info { + stream_id: "mo0b6mh9" +} + +2025-01-18 19:54:58,445 DEBUG HandlerThread:2566524 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 19:54:58,452 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/x1yqpgel-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:54:58,483 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/wt9sjipv-code/FISH/eval_robot.py +2025-01-18 19:54:58,498 DEBUG SenderThread:2566524 [sender.py:send():379] send: telemetry +2025-01-18 19:54:58,498 DEBUG SenderThread:2566524 [sender.py:send():379] send: tbrecord +2025-01-18 19:54:58,499 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:54:58,499 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 with policy live +2025-01-18 19:54:58,685 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/2ud0oelk-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:54:58,950 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:54:58,950 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/diff.patch +2025-01-18 19:54:58,950 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-metadata.json +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code/FISH/eval_robot.py +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/requirements.txt +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code/FISH +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code +2025-01-18 19:54:59,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:00,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:00,949 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:01,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:02,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:02,836 DEBUG SenderThread:2566524 [sender.py:send():379] send: config +2025-01-18 19:55:02,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:03,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:04,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:04,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:05,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:06,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:07,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:08,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:08,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:08,952 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:09,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:10,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:11,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:12,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:12,953 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:13,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:55:13,403 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:55:13,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:13,497 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:14,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:15,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:16,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:17,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:18,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:19,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:19,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:20,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:21,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:22,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:23,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:24,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:25,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:25,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:26,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:27,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:28,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:55:28,403 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:55:28,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:29,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:30,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:30,558 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:30,958 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml +2025-01-18 19:55:31,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:32,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:33,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:34,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:35,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:36,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:36,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:37,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:38,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:39,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:40,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:41,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:41,716 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:42,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:42,961 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:43,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:55:43,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:55:43,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:44,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:44,962 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:45,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:46,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:46,963 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:47,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:47,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:48,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:49,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:50,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:51,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:52,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:53,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:53,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:54,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:55,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:56,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:57,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:58,049 DEBUG SystemMonitor:2566524 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 19:55:58,049 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:55:58,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:55:58,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:55:58,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:58,463 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:59,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:00,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:01,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:02,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:03,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:04,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:04,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:05,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:06,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:07,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:08,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:09,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:09,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:10,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:11,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:12,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:13,362 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:56:13,363 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 with policy now +2025-01-18 19:56:13,363 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:56:13,364 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:56:13,365 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:56:13,365 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:56:13,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:56:13,405 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:56:13,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:13,622 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/c33jqz39-media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media +2025-01-18 19:56:14,242 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/5q0i49hy-media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 +2025-01-18 19:56:14,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:14,506 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:14,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:15,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:16,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:17,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:18,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:18,972 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:19,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:19,590 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:20,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:20,972 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:21,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:21,973 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:56:22,218 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/vqv8y13n-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:56:22,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:22,973 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:23,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:24,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:24,974 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:25,339 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:25,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:26,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:26,975 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:27,417 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:28,081 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:56:28,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:56:28,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:56:28,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:29,417 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:30,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:30,526 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:31,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:32,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:33,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:34,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:35,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:36,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:36,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:37,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:38,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:39,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:40,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:41,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:42,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:42,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:43,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:56:43,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:56:43,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:44,419 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:45,419 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:46,419 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:47,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:47,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:48,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:49,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:50,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:51,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:52,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:53,098 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:53,351 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:56:53,352 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 with policy now +2025-01-18 19:56:53,354 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:56:53,355 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:56:53,355 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:56:53,356 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:56:53,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:53,621 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/cj77boio-media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 +2025-01-18 19:56:53,984 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:56:53,984 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 +2025-01-18 19:56:53,984 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:56:54,253 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/bkkpb6kx-media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 +2025-01-18 19:56:54,423 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:54,984 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:55,425 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:56,423 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:57,440 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:58,050 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:56:58,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:56:58,409 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:56:58,453 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:58,565 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:58,985 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:59,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:00,431 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:00,986 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:01,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:02,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:03,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:03,634 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:03,860 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:57:03,860 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:57:03,861 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:57:03,987 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:57:03,987 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:57:04,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:04,987 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:05,431 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:06,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:06,988 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:07,438 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:08,432 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:08,989 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:09,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:09,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:10,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:11,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:12,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:13,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:57:13,405 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:57:13,449 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:14,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:14,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:15,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:16,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:17,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:18,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:19,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:20,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:20,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:21,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:22,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:23,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:24,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:25,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:26,315 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:26,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:27,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:28,051 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:57:28,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:57:28,406 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:57:28,453 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:28,996 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:29,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:30,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:31,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:31,538 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:32,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:33,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:34,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:35,439 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:36,051 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:57:36,051 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 with policy now +2025-01-18 19:57:36,056 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:57:36,056 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:57:36,057 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:57:36,057 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:57:36,312 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/3nrtrf6b-media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 +2025-01-18 19:57:36,438 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:36,998 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:36,998 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:57:36,999 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 +2025-01-18 19:57:36,999 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:57:37,069 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:37,284 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/1puhry0r-media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 +2025-01-18 19:57:37,440 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:38,439 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:38,999 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:39,451 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:40,440 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:41,000 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:41,448 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:42,446 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:42,483 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:43,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:57:43,408 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:57:43,446 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:44,457 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:45,001 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:45,451 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:46,448 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:47,447 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:48,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:48,450 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:49,456 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:50,470 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:50,701 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:57:50,701 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:57:50,701 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:57:51,003 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:51,003 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:57:51,004 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:57:51,303 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/p5shhvgr-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:57:51,452 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:52,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:53,004 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:53,421 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:53,464 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:54,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:55,004 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:55,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:56,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:57,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:58,051 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:57:58,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:57:58,406 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:57:58,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:58,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:59,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:00,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:01,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:02,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:03,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:04,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:04,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:05,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:06,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:07,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:08,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:09,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:09,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:10,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:11,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:12,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:13,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:58:13,406 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:58:13,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:14,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:14,945 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:15,011 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:15,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:16,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:17,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:18,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:19,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:20,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:20,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:21,485 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:22,320 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:58:22,320 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 with policy now +2025-01-18 19:58:22,323 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:58:22,323 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:58:22,323 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:58:22,324 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:58:22,461 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:22,571 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/on9jey56-media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 +2025-01-18 19:58:23,014 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:23,014 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:58:23,014 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 +2025-01-18 19:58:23,014 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:58:23,303 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ih02u9zo-media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 +2025-01-18 19:58:23,467 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:24,479 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:25,015 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:25,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:25,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:26,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:27,015 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:27,486 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:28,052 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:58:28,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:58:28,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:58:28,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:29,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:30,502 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:30,881 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:31,017 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:31,498 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:32,017 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:58:32,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:32,751 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:58:32,751 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:58:32,751 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:58:33,017 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:33,018 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:58:33,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:34,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:35,018 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:35,500 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:35,921 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:36,496 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:37,019 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:37,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:38,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:39,020 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:39,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:40,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:41,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:41,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:42,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:43,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:58:43,414 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:58:43,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:44,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:45,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:46,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:46,517 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:47,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:48,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:49,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:50,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:51,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:52,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:52,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:53,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:54,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:55,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:56,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:57,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:57,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:58,052 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:58:58,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:58:58,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:58:58,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:59,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:00,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:01,492 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:02,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:02,492 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:03,492 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:04,504 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:04,665 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:59:04,665 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 with policy now +2025-01-18 19:59:04,673 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:59:04,674 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:59:04,674 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:59:04,675 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:59:04,936 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ibqypos3-media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 +2025-01-18 19:59:05,029 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:05,029 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:59:05,029 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 +2025-01-18 19:59:05,029 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:59:05,299 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/b8yu4iz5-media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 +2025-01-18 19:59:05,499 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:06,508 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:07,030 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:07,511 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:07,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:08,508 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:09,504 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:10,507 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:11,031 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:11,503 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:12,508 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:13,032 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:13,289 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:13,438 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:59:13,438 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:59:13,525 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:14,522 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:15,509 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:16,520 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:17,524 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:18,441 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:18,524 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:19,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:20,517 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:21,521 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:22,532 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:22,810 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:59:22,810 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:59:22,810 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:59:23,035 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:59:23,035 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:59:23,290 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/rhopy0zc-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:59:23,532 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:23,868 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:24,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:25,036 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:25,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:26,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:27,532 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:28,052 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:59:28,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:59:28,414 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:59:28,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:29,037 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:29,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:29,567 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:30,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:31,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:32,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:33,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:34,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:35,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:35,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:36,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:37,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:38,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:39,041 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:39,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:40,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:40,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:41,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:42,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:43,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:59:43,414 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:59:43,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:44,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:45,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:45,550 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:46,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:47,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:48,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:49,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:50,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:51,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:51,536 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:52,536 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:53,536 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:54,541 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:55,047 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:55,106 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:59:55,106 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 with policy now +2025-01-18 19:59:55,118 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:59:55,119 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:59:55,119 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:59:55,119 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:59:55,380 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/4idcojoq-media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 +2025-01-18 19:59:55,541 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:56,047 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:59:56,047 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 +2025-01-18 19:59:56,048 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:59:56,349 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/8k3jrrr9-media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 +2025-01-18 19:59:56,538 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:57,048 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:57,175 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:57,548 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:58,053 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:59:58,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:59:58,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:59:58,540 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:59,549 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:00,556 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:01,049 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:01,569 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:02,556 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:02,761 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:03,050 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:03,559 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:04,558 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:04,861 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:00:04,861 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:00:04,861 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:00:05,051 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:00:05,051 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:00:05,560 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:06,576 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:07,051 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:07,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:07,925 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:08,567 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:09,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:10,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:11,053 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:11,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:12,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:13,239 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:13,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:00:13,414 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:00:13,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:14,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:15,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:16,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:17,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:18,429 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:18,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:19,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:20,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:21,056 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:21,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:22,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:23,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:23,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:24,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:25,058 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:25,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:26,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:27,059 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:27,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:28,053 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:00:28,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:00:28,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:00:28,532 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:28,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:29,060 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:29,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:30,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:31,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:32,564 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:33,061 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:33,564 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:34,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:34,564 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:35,564 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:36,581 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:37,063 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:37,570 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:37,641 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:00:37,641 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:00:37,642 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 with policy now +2025-01-18 20:00:37,642 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:00:37,642 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:00:37,642 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:00:37,927 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/gpirp6zb-media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 +2025-01-18 20:00:38,063 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:00:38,063 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 +2025-01-18 20:00:38,063 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:00:38,337 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/epmkbwxg-media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 +2025-01-18 20:00:38,581 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:39,064 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:39,439 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:39,573 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:40,583 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:41,586 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:42,591 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:43,065 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:43,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:00:43,430 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:00:43,581 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:44,575 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:45,066 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:45,353 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:45,591 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:46,590 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:47,585 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:48,590 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:48,920 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:00:48,920 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:00:48,921 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:00:49,068 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:49,068 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:00:49,068 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:00:49,321 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/fo54zghb-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:00:49,593 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:50,377 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:50,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:51,069 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:51,596 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:52,597 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:53,069 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:53,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:54,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:55,451 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:55,602 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:56,597 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:57,071 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:57,597 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:58,054 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:00:58,423 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:00:58,423 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:00:58,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:59,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:00,578 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:00,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:01,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:02,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:03,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:04,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:05,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:06,437 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:06,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:07,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:08,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:09,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:10,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:11,437 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:11,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:12,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:13,423 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:01:13,423 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:01:13,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:14,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:15,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:16,558 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:16,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:17,600 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:18,600 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:19,602 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:20,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:21,080 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:21,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:22,161 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:22,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:23,628 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:24,206 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:01:24,206 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 with policy now +2025-01-18 20:01:24,210 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:01:24,210 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:01:24,210 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:01:24,210 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:01:24,461 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/845kbyic-media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 +2025-01-18 20:01:24,603 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:25,081 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:25,081 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:01:25,082 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 +2025-01-18 20:01:25,082 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:01:25,403 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/x9ppir5n-media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 +2025-01-18 20:01:25,630 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:26,631 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:27,082 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:27,255 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:27,644 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:28,054 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:01:28,432 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:01:28,432 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:01:28,637 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:29,083 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:29,632 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:30,641 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:31,642 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:32,653 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:32,853 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:33,084 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:33,644 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:34,643 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:34,977 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:01:34,978 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:01:34,978 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:01:35,085 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:01:35,085 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:01:35,656 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:36,658 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:37,086 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:37,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:37,877 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:38,671 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:39,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:40,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:41,088 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:41,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:42,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:43,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:43,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:01:43,431 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:01:43,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:44,646 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:45,646 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:46,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:47,646 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:48,444 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:48,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:49,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:50,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:51,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:52,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:53,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:53,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:54,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:55,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:56,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:57,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:58,055 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:01:58,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:01:58,431 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:01:58,585 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:58,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:59,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:00,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:01,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:02,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:03,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:04,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:04,649 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:05,649 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:06,665 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:07,097 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:07,217 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:02:07,217 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 with policy now +2025-01-18 20:02:07,228 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:02:07,228 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:02:07,229 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:02:07,229 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:02:07,469 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/wttdunw1-media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 +2025-01-18 20:02:07,662 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:08,098 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:02:08,098 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 +2025-01-18 20:02:08,098 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:02:08,370 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/4uqktciz-media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 +2025-01-18 20:02:08,662 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:09,098 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:09,460 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:09,666 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:10,679 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:11,689 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:12,689 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:13,100 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:13,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:02:13,455 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:02:13,697 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:14,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:15,033 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:15,101 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:15,697 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:16,722 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:17,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:18,029 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:02:18,029 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:02:18,029 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:02:18,102 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:02:18,102 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:02:18,383 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/rjc30w1s-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:02:18,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:19,102 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:19,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:20,727 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:20,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:21,103 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:21,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:22,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:23,104 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:23,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:24,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:25,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:26,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:26,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:27,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:28,055 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:02:28,432 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:02:28,432 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:02:28,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:29,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:30,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:31,503 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:31,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:32,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:33,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:34,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:35,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:36,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:37,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:37,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:38,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:39,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:40,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:41,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:42,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:42,701 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:43,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:02:43,433 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:02:43,701 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:44,701 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:45,701 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:46,702 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:47,505 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:47,702 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:48,702 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:49,725 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:50,381 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:02:50,381 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 with policy now +2025-01-18 20:02:50,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:02:50,415 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:02:50,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:02:50,415 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:02:50,671 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ikkwxibk-media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 +2025-01-18 20:02:50,724 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:51,115 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:51,115 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:02:51,115 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 +2025-01-18 20:02:51,115 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:02:51,402 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/jhlb5rn9-media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 +2025-01-18 20:02:51,725 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:52,726 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:53,116 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:53,474 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:53,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:54,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:55,116 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:55,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:56,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:57,735 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:58,056 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:02:58,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:02:58,445 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:02:58,579 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:58,754 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:59,118 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:59,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:00,753 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:01,082 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:03:01,082 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:03:01,083 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:03:01,119 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:03:01,119 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:03:01,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:02,756 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:03,119 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:03,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:04,499 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:04,754 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:05,774 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:06,764 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:07,121 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:07,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:08,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:09,510 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:09,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:10,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:11,123 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:11,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:12,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:13,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:03:13,433 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:03:13,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:14,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:14,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:15,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:16,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:17,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:18,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:19,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:20,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:20,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:21,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:22,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:23,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:24,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:25,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:25,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:26,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:27,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:28,056 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:03:28,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:03:28,435 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:03:28,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:29,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:30,506 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:30,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:31,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:32,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:33,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:34,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:35,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:36,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:36,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:37,776 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:38,111 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:03:38,111 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 with policy now +2025-01-18 20:03:38,119 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:03:38,119 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:03:38,119 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:03:38,120 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:03:38,133 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:03:38,134 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 +2025-01-18 20:03:38,134 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:03:38,379 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/silop977-media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 +2025-01-18 20:03:38,642 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/soaimu9s-media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 +2025-01-18 20:03:38,768 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:39,134 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:39,769 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:40,803 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:41,135 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:41,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:41,789 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:42,790 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:43,135 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:43,469 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:03:43,469 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:03:43,797 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:44,798 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:45,816 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:46,845 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:46,937 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:47,137 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:47,821 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:48,803 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:49,796 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:50,140 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:03:50,141 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:03:50,141 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:03:50,142 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:03:50,142 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:03:50,384 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ncdu7a72-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:03:50,855 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:51,142 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:51,842 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:52,449 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:52,813 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:53,822 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:54,845 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:55,144 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:55,842 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:56,841 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:57,565 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:57,854 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:58,057 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:03:58,470 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:03:58,470 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:03:58,838 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:59,840 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:00,829 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:01,146 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:01,829 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:02,829 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:03,097 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:03,829 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:04,830 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:05,830 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:06,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:07,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:08,516 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:08,832 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:09,833 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:10,833 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:11,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:12,833 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:13,438 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:04:13,438 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:04:13,516 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:13,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:14,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:15,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:16,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:17,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:18,516 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:18,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:19,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:20,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:21,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:22,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:23,517 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:23,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:24,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:25,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:26,837 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:27,857 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:28,057 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:04:28,394 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:04:28,394 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 with policy now +2025-01-18 20:04:28,417 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:04:28,417 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:04:28,417 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:04:28,418 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:04:28,470 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:04:28,470 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:04:28,538 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:28,661 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/447rwsgw-media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 +2025-01-18 20:04:28,859 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:29,157 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:29,157 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:04:29,157 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 +2025-01-18 20:04:29,157 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:04:29,452 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/9vv93n32-media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 +2025-01-18 20:04:29,848 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:30,857 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:31,158 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:31,851 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:32,874 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:33,158 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:33,555 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:33,854 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:34,863 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:35,855 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:36,886 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:37,160 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:37,877 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:38,581 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:38,884 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:39,161 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:39,161 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:04:39,200 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:04:39,200 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:04:39,200 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:04:39,932 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:40,161 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:04:40,888 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:41,161 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:41,917 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:42,878 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:43,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:04:43,445 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:04:43,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:44,579 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:44,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:45,163 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:45,878 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:46,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:47,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:48,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:49,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:50,519 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:50,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:51,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:52,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:53,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:54,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:55,519 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:55,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:56,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:57,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:58,058 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:04:58,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:04:58,445 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:04:58,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:59,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:00,582 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:00,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:01,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:02,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:03,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:04,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:05,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:06,520 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:06,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:07,881 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:08,881 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:09,932 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:10,907 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:11,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:11,296 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:05:11,296 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 with policy now +2025-01-18 20:05:11,301 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:05:11,301 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:05:11,302 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:05:11,302 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:05:11,554 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/os7081uj-media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 +2025-01-18 20:05:11,918 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:12,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:05:12,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 +2025-01-18 20:05:12,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:05:12,385 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:12,458 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/86r0qhw1-media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 +2025-01-18 20:05:12,929 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:13,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:13,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:05:13,482 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:05:13,909 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:14,915 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:15,966 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:16,922 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:17,176 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:17,634 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:17,928 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:18,938 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:19,177 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:19,950 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:20,956 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:21,943 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:22,968 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:23,536 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:23,964 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:24,973 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:25,971 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:26,971 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:27,976 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:28,058 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:05:28,495 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:05:28,496 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:05:28,650 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:28,996 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:29,964 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:30,982 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:31,969 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:32,981 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:34,000 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:34,551 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:34,998 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:35,971 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:36,994 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:38,007 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:38,975 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:39,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:39,997 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:40,991 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:41,992 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:43,029 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:43,471 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:05:43,472 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:05:43,999 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:44,801 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:45,006 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:46,040 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:47,019 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:48,018 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:49,033 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:50,031 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:50,572 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:51,044 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:52,048 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:53,050 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:54,078 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:55,083 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:55,578 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:56,070 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:57,085 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:57,192 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:57,192 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:05:57,298 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:05:57,298 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:05:57,298 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:05:58,058 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:05:58,081 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:58,192 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:05:58,480 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:05:58,480 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:05:59,085 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:59,193 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:00,090 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:00,849 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:01,083 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:02,140 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:03,092 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:03,194 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:04,113 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:05,112 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:06,114 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:06,579 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:07,112 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:08,130 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:09,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:10,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:11,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:11,197 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:12,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:12,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:13,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:13,474 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:06:13,474 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:06:14,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:15,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:16,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:17,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:17,579 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:18,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:19,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:20,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:21,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:22,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:23,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:23,572 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:24,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:25,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:26,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:27,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:28,059 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:06:28,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:28,474 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:06:28,474 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:06:29,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:29,572 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:30,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:31,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:32,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:33,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:34,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:34,572 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:35,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:36,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:37,133 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:37,209 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:37,521 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:06:37,521 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 with policy now +2025-01-18 20:06:37,545 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:06:37,546 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:06:37,546 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:06:37,546 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:06:37,782 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/gm0yq7b0-media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 +2025-01-18 20:06:38,124 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:38,210 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:06:38,210 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 +2025-01-18 20:06:38,210 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:06:38,499 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/26qywfdv-media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 +2025-01-18 20:06:39,137 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:39,210 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:39,588 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:40,150 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:41,136 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:42,153 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:43,146 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:43,212 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:43,492 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:06:43,492 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:06:44,164 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:45,153 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:45,212 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:45,497 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:46,171 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:47,151 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:48,176 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:49,146 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:49,361 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:06:49,361 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:06:49,366 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:06:50,170 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:50,214 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:06:50,215 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:06:50,469 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/x6a11xcl-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:06:50,550 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:51,174 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:51,215 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:52,183 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:53,188 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:54,172 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:55,175 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:55,217 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:55,771 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:56,171 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:57,172 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:58,059 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:06:58,174 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:58,496 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:06:58,497 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:06:59,190 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:00,221 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:01,202 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:01,617 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:02,201 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:03,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:03,220 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:04,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:05,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:06,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:07,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:07,594 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:08,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:09,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:10,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:11,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:12,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:13,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:13,224 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:13,373 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:13,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:07:13,483 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:07:14,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:15,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:16,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:17,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:18,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:18,594 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:19,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:20,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:21,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:22,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:23,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:24,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:24,594 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:25,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:26,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:27,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:28,060 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:07:28,200 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:28,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:07:28,483 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:07:29,203 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:30,230 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:30,305 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:30,978 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:07:30,978 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 with policy now +2025-01-18 20:07:30,998 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:07:30,999 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:07:30,999 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:07:30,999 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:07:31,215 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:31,232 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:31,232 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:07:31,232 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 +2025-01-18 20:07:31,233 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:07:31,250 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ojwd1tbi-media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 +2025-01-18 20:07:31,517 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/fp2c2bo7-media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 +2025-01-18 20:07:32,235 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:33,233 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:33,234 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:34,233 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:34,275 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:35,262 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:35,713 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:36,234 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:36,248 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:37,271 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:38,233 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:39,265 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:40,237 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:40,237 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:07:40,237 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:40,426 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:07:40,426 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:07:40,426 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:07:41,238 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:07:41,238 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:07:41,259 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:41,353 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:42,239 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:42,244 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:43,261 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:43,504 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:07:43,504 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:07:44,239 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:44,294 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:45,293 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:46,240 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:46,314 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:46,686 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:47,301 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:48,292 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:49,298 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:50,303 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:51,293 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:52,306 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:52,643 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:53,291 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:54,336 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:55,281 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:56,244 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:56,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:57,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:58,060 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:07:58,061 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:58,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:58,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:07:58,488 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:07:59,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:00,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:01,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:02,247 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:02,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:03,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:03,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:04,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:05,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:06,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:07,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:08,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:08,616 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:09,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:10,250 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:10,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:11,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:12,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:13,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:13,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:08:13,488 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:08:14,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:14,616 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:15,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:16,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:17,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:18,285 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:19,285 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:19,616 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:20,285 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:21,285 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:22,290 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:22,534 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:08:22,535 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 with policy now +2025-01-18 20:08:22,554 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:08:22,554 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:08:22,554 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:08:22,555 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:08:22,807 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/9oi6gffe-media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 +2025-01-18 20:08:23,256 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:08:23,256 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 +2025-01-18 20:08:23,256 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:08:23,343 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:23,596 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/zjpz506b-media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 +2025-01-18 20:08:24,256 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:24,301 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:24,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:25,257 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:25,339 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:26,325 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:27,329 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:28,061 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:08:28,258 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:28,316 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:28,556 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:08:28,556 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:08:29,355 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:29,669 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:30,354 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:31,357 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:32,260 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:32,383 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:32,487 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:08:32,488 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:08:32,488 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:08:33,260 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:08:33,260 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:08:33,362 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:34,261 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:34,348 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:34,689 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:35,390 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:36,261 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:36,357 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:37,393 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:38,262 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:38,377 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:39,377 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:40,263 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:40,387 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:40,644 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:41,389 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:42,392 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:43,419 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:43,540 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:08:43,540 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:08:44,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:45,428 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:45,678 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:46,431 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:47,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:48,267 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:48,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:49,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:50,267 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:50,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:51,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:51,623 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:52,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:53,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:54,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:55,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:56,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:56,623 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:57,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:58,061 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:08:58,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:58,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:08:58,501 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:08:59,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:00,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:01,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:02,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:02,623 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:03,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:04,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:05,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:06,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:07,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:07,623 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:08,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:09,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:10,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:11,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:12,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:13,086 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:13,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:13,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:09:13,501 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:09:14,278 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:14,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:15,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:15,798 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:09:15,798 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 with policy now +2025-01-18 20:09:15,802 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:09:15,802 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:09:15,802 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:09:15,802 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:09:16,073 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/dxyaze9m-media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 +2025-01-18 20:09:16,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:16,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:09:16,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 +2025-01-18 20:09:16,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:09:16,453 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:16,551 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/s8tnjemt-media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 +2025-01-18 20:09:17,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:17,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:18,280 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:18,478 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:18,663 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:19,443 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:20,452 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:21,477 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:22,282 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:22,462 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:23,465 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:23,665 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:24,282 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:24,450 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:25,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:26,471 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:27,284 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:09:27,452 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:27,541 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/53vvgyah-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:09:27,555 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:09:27,555 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:09:27,555 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:09:28,061 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:09:28,284 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:28,284 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:09:28,483 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:28,545 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:09:28,545 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:09:28,675 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:29,470 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:30,285 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:30,543 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:31,541 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:32,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:33,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:34,287 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:34,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:34,663 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:35,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:36,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:37,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:38,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:39,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:39,664 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:40,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:41,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:42,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:43,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:43,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:09:43,501 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:09:44,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:45,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:45,664 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:46,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:47,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:48,293 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:48,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:49,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:50,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:50,664 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:51,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:52,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:53,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:54,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:55,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:55,664 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:56,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:57,493 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:58,062 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:09:58,298 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:58,493 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:58,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:09:58,502 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:09:59,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:00,299 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:00,299 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 +2025-01-18 20:10:00,299 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:10:00,338 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:10:00,338 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 with policy now +2025-01-18 20:10:00,362 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:10:00,362 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:10:00,362 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:10:00,363 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:10:00,502 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:00,565 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/4vutdphb-media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 +2025-01-18 20:10:00,849 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/nmxvdykm-media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 +2025-01-18 20:10:01,299 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:10:01,476 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:01,524 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:02,300 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:02,539 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:03,300 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:03,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:04,586 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:05,551 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:06,302 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:06,551 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:07,181 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:07,578 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:08,602 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:09,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:09,606 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:10:09,606 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:10:09,607 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:10:10,303 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:10,304 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:10:10,304 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:10:10,611 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:11,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:12,304 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:12,629 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:12,661 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:13,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:10:13,536 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:10:13,640 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:14,607 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:15,607 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:16,306 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:16,607 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:17,607 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:18,209 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:18,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:19,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:20,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:21,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:22,309 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:22,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:23,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:23,680 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:24,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:25,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:26,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:27,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:28,062 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:10:28,506 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:10:28,507 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:10:28,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:28,680 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:29,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:30,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:31,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:32,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:33,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:33,680 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:34,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:35,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:36,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:37,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:38,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:38,680 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:39,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:40,611 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:41,654 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:42,318 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:42,613 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:10:42,613 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 with policy now +2025-01-18 20:10:42,622 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:42,669 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:10:42,669 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:10:42,670 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:10:42,670 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:10:42,863 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/w2hj5yjy-media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 +2025-01-18 20:10:43,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:43,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:10:43,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 +2025-01-18 20:10:43,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:10:43,541 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:10:43,541 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:10:43,604 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/hh67nec3-media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 +2025-01-18 20:10:43,622 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:43,722 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:44,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:44,659 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:45,320 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:45,662 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:46,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:47,642 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:48,321 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:48,653 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:48,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:49,674 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:50,703 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:51,688 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:52,323 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:52,685 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:53,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:54,720 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:54,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:55,735 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:56,664 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:10:56,664 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:10:56,664 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:10:56,714 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:57,325 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:10:57,325 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:10:57,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:58,063 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:10:58,325 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:58,542 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:10:58,542 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:10:58,715 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:59,719 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:00,326 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:00,719 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:00,719 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:01,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:02,328 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:02,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:03,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:04,329 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:04,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:05,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:06,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:06,710 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:07,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:08,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:09,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:10,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:11,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:11,710 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:12,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:13,508 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:11:13,509 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:11:13,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:14,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:15,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:16,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:16,710 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:17,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:18,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:19,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:20,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:21,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:21,711 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:22,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:23,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:24,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:25,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:26,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:26,711 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:27,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:28,063 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:11:28,546 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:11:28,546 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:11:28,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:29,340 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:11:29,340 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 with policy now +2025-01-18 20:11:29,343 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 +2025-01-18 20:11:29,343 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:11:29,348 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:11:29,348 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:11:29,348 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:29,349 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:29,598 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/q9wwsyvm-media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 +2025-01-18 20:11:29,716 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:29,856 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/8dqa5vaf-media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 +2025-01-18 20:11:30,343 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:30,343 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:11:30,722 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:31,718 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:32,344 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:32,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:32,774 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:33,737 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:34,346 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:34,723 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:35,762 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:36,772 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:37,523 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:37,763 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:38,347 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:38,764 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:39,348 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:11:39,720 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:11:39,720 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:39,721 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:39,804 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:40,348 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:40,349 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:11:40,522 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 20:11:40,522 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 20:11:40,717 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:11:40,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:41,012 INFO wandb-upload_3:2566524 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu0o677tu +2025-01-18 20:11:41,074 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 20:11:41,085 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:11:41,349 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:11:41,349 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json +2025-01-18 20:11:41,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json +2025-01-18 20:11:41,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media +2025-01-18 20:11:41,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval +2025-01-18 20:11:41,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table +2025-01-18 20:11:41,402 INFO SenderThread:2566524 [sender.py:send_request_log_artifact():1525] logged artifact run-mo0b6mh9-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTk1Mzc3Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA2Mzk2Nw==', 'latestArtifact': None}} +2025-01-18 20:11:41,402 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:11:41,403 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json with policy now +2025-01-18 20:11:41,403 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:11:41,403 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,403 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,404 DEBUG SenderThread:2566524 [sender.py:send():379] send: telemetry +2025-01-18 20:11:41,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,407 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,408 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,408 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,408 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,409 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,409 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,409 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,409 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,410 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,410 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,410 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,410 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,411 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,411 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,411 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,412 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,412 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,412 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,412 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,413 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,413 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,413 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 20:11:41,623 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/yzdnup9g-media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json +2025-01-18 20:11:41,644 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/24g65024-media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json +2025-01-18 20:11:41,661 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/sr481jlx-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:11:41,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:41,782 INFO wandb-upload_3:2566524 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe6tbkd_4 +2025-01-18 20:11:41,871 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/xi840sew-media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json +2025-01-18 20:11:42,179 INFO SenderThread:2566524 [sender.py:send_request_log_artifact():1525] logged artifact run-mo0b6mh9-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTk1Mzc4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA2Mzk2OA==', 'latestArtifact': None}} +2025-01-18 20:11:42,179 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:11:42,179 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json with policy now +2025-01-18 20:11:42,180 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:11:42,180 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:42,180 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:42,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:11:42,428 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/44lpp5dk-media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json +2025-01-18 20:11:42,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:43,185 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:43,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml +2025-01-18 20:11:43,513 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:11:43,513 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:11:43,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:44,351 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:44,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:45,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:46,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:47,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:48,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:48,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:49,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:50,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:51,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:52,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:53,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:53,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:54,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:55,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:56,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:57,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:58,064 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:11:58,513 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:11:58,513 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:11:58,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:58,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:59,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:00,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:01,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:02,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:03,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:04,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:04,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:05,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:06,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:07,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:08,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:09,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:09,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:10,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:11,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:12,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:13,513 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:12:13,513 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:12:13,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:14,021 DEBUG SenderThread:2566524 [sender.py:send():379] send: exit +2025-01-18 20:12:14,021 INFO SenderThread:2566524 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 20:12:14,021 INFO SenderThread:2566524 [sender.py:send_exit():588] handling runtime: 1035 +2025-01-18 20:12:14,022 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:12:14,022 INFO SenderThread:2566524 [sender.py:send_exit():594] send defer +2025-01-18 20:12:14,022 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,022 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 20:12:14,022 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:14,022 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 20:12:14,022 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 1 +2025-01-18 20:12:14,022 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,023 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 20:12:14,023 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:14,023 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 20:12:14,023 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 2 +2025-01-18 20:12:14,023 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,023 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 20:12:14,023 INFO HandlerThread:2566524 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 20:12:14,023 DEBUG SystemMonitor:2566524 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 20:12:14,023 DEBUG SystemMonitor:2566524 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 20:12:14,023 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 20:12:14,024 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 20:12:14,074 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 20:12:14,074 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 20:12:14,074 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined network monitor +2025-01-18 20:12:14,074 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:14,074 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 20:12:14,074 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 3 +2025-01-18 20:12:14,075 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:12:14,075 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,075 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 20:12:14,075 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:14,075 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 20:12:14,075 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 4 +2025-01-18 20:12:14,075 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,075 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 20:12:14,366 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:12:16,367 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:12:20,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:20,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:20,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 20:12:20,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:20,731 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:20,731 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 20:12:20,731 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 5 +2025-01-18 20:12:20,731 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:20,732 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:20,732 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 20:12:20,740 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:12:20,740 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:12:20,740 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:12:20,741 DEBUG SenderThread:2566524 [sender.py:send():379] send: summary +2025-01-18 20:12:20,741 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:12:20,741 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:20,741 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 20:12:20,741 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 6 +2025-01-18 20:12:20,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:20,741 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 20:12:20,742 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:20,742 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 20:12:20,742 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 7 +2025-01-18 20:12:20,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:20,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:20,742 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 20:12:20,742 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:20,742 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 20:12:21,022 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:21,370 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:12:22,841 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 8 +2025-01-18 20:12:22,841 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:22,842 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:22,842 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 20:12:22,842 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:22,842 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 20:12:22,842 INFO SenderThread:2566524 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 20:12:22,845 INFO SenderThread:2566524 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 20:12:22,846 INFO SenderThread:2566524 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 20:12:22,861 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 9 +2025-01-18 20:12:22,861 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:22,861 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 20:12:22,862 DEBUG SenderThread:2566524 [sender.py:send():379] send: artifact +2025-01-18 20:12:23,023 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:23,239 INFO wandb-upload_2:2566524 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe9xdfu6j +2025-01-18 20:12:23,239 INFO wandb-upload_0:2566524 [upload_job.py:push():85] Skipped uploading /tmp/tmp6zpygqk6/wandb-job.json +2025-01-18 20:12:23,359 INFO wandb-upload_3:2566524 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpi1brqpjv +2025-01-18 20:12:23,371 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:12:23,938 INFO SenderThread:2566524 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTk1NDMyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTkxOTkxNA==', 'versionIndex': 448}}} +2025-01-18 20:12:23,938 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:23,938 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 20:12:23,938 INFO SenderThread:2566524 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 20:12:24,373 INFO SenderThread:2566524 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files +2025-01-18 20:12:24,373 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml config.yaml +2025-01-18 20:12:24,373 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:12:24,373 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json wandb-summary.json +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/requirements.txt requirements.txt +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-metadata.json wandb-metadata.json +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log output.log +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/diff.patch diff.patch +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 10 +2025-01-18 20:12:24,382 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:24,382 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:24,382 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 20:12:24,382 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:24,382 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 20:12:24,382 INFO SenderThread:2566524 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 20:12:24,566 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml +2025-01-18 20:12:24,581 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:12:24,583 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/requirements.txt +2025-01-18 20:12:24,589 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:12:24,789 INFO Thread-11 (_thread_body):2566524 [sender.py:transition_state():614] send defer: 11 +2025-01-18 20:12:24,789 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:24,789 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 20:12:24,790 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:24,790 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 20:12:24,790 INFO SenderThread:2566524 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 20:12:24,793 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 12 +2025-01-18 20:12:24,793 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:24,793 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 20:12:24,793 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:24,793 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 20:12:24,793 INFO SenderThread:2566524 [file_stream.py:finish():601] file stream finish called +2025-01-18 20:12:25,014 INFO SenderThread:2566524 [file_stream.py:finish():605] file stream finish is done +2025-01-18 20:12:25,014 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 13 +2025-01-18 20:12:25,014 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:25,014 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 20:12:25,015 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:25,015 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 20:12:25,015 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 14 +2025-01-18 20:12:25,015 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:25,015 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 20:12:25,015 DEBUG SenderThread:2566524 [sender.py:send():379] send: final +2025-01-18 20:12:25,015 DEBUG SenderThread:2566524 [sender.py:send():379] send: footer +2025-01-18 20:12:25,015 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:25,015 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 20:12:25,016 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:25,016 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 20:12:25,016 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:25,016 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:25,016 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: server_info +2025-01-18 20:12:25,017 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 20:12:25,018 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 20:12:25,019 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:25,078 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:25,078 INFO MainThread:2566524 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 20:12:25,079 INFO MainThread:2566524 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 20:12:25,079 INFO MainThread:2566524 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 20:12:25,080 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 20:12:25,080 INFO HandlerThread:2566524 [handler.py:finish():882] shutting down handler +2025-01-18 20:12:26,017 INFO WriterThread:2566524 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/run-mo0b6mh9.wandb +2025-01-18 20:12:26,078 INFO SenderThread:2566524 [sender.py:finish():1615] shutting down sender +2025-01-18 20:12:26,079 INFO SenderThread:2566524 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 20:12:26,079 INFO SenderThread:2566524 [file_pusher.py:join():175] waiting for file pusher diff --git a/mo0b6mh9/wandb/debug.log b/mo0b6mh9/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8969ed10d7699ba2a7f1de0c4eb48e72a38a10ef --- /dev/null +++ b/mo0b6mh9/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Configure stats pid to 2566358 +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/settings +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug.log +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug-internal.log +2025-01-18 19:54:57,682 INFO MainThread:2566358 [wandb_init.py:init():569] calling init triggers +2025-01-18 19:54:57,682 INFO MainThread:2566358 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9'} +2025-01-18 19:54:57,682 INFO MainThread:2566358 [wandb_init.py:init():619] starting backend +2025-01-18 19:54:57,682 INFO MainThread:2566358 [wandb_init.py:init():623] setting up manager +2025-01-18 19:54:57,687 INFO MainThread:2566358 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 19:54:57,688 INFO MainThread:2566358 [wandb_init.py:init():631] backend started and connected +2025-01-18 19:54:57,700 INFO MainThread:2566358 [wandb_init.py:init():720] updated telemetry +2025-01-18 19:54:57,707 INFO MainThread:2566358 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 19:54:57,962 INFO MainThread:2566358 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 19:54:58,032 INFO MainThread:2566358 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 19:54:58,032 INFO MainThread:2566358 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 19:54:58,403 INFO MainThread:2566358 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 19:54:58,403 INFO MainThread:2566358 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 19:54:58,403 INFO MainThread:2566358 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 19:54:58,404 INFO MainThread:2566358 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 19:54:58,405 INFO MainThread:2566358 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 19:54:58,405 INFO MainThread:2566358 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/tb, True +2025-01-18 19:55:02,836 INFO MainThread:2566358 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 20:12:26,080 WARNING MsgRouterThr:2566358 [router.py:message_loop():77] message_loop has been closed diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code/FISH/eval_robot.py b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1f44a25820ed7f0e4e3afa7da2522623d275076 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3465_0' +wandb_notes: + desc: null + value: 3465_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737248097 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/diff.patch b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..f434f8847b435a60c5fa887dbb28833036f5dd51 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/diff.patch @@ -0,0 +1,102 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..03918f5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -132,8 +132,18 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++wandb_run_id: '3465_0' ++# wandb_run_id: '3465_1' # seed 1 ++# wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +@@ -191,7 +201,7 @@ wandb_run_id: '1002_0' + true_action_history: false + wandb_notes: null + +-checkpoint_epoch: 12000 # for depth + all contact ++checkpoint_epoch: 7500 # for depth + all contact + load_residual_weight: false + + checkpoint_root_dir: '/home/${oc.env:USER}/fish_leon/FISH' +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..9509617 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 7500 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json new file mode 100644 index 0000000000000000000000000000000000000000..933324908d635bf48deaa7e225d713ceba65d71b --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.05, 1, 1, 1, 1, 1, 1, 1, 1]]} \ No newline at end of file diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json new file mode 100644 index 0000000000000000000000000000000000000000..2d3cd6665c510044ec1dbe0fae803012ac0cba0e --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[21, 20, 1.05, 1, 1, 1, 1, 1, 1, 1, 1]]} \ No newline at end of file diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7720b7930a725afa338b6988b7ac911aff3407aa --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd640c135955b3d54fec7ae6fb41c2edef3b17fda1044801e3101daccb7fcae9 +size 1185117 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da10d4e37e93db6ad5c26cea42a67152c526e7ee --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:708e5d86baaf5c876d21100f22ad7b92231cbe37549a12b676e35762ac00de0b +size 1369725 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e3e6216017c05cf3cd57ad8d7d04c09f2f8f07e7 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e48fceb469367c5942f582757e64e07e7cccfbdf6b03d7970d1ccbe00037983 +size 1401639 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4273b35b4910d6e3b36f6b053c5355e8e5bc8506 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a56ddf9bcd287e59382b84cd2c72971de6aaa276cbfe72b32f05001e3a2a734 +size 1170389 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2150d40d7f69f604f79712478a8c3b2eed4d2859 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce205101458cc00950f123e1c7e065c5f0df37ab903d9ab26f3e942715d9d157 +size 1488237 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15cb9b16a42f7db710b5f755c0592255b292ac7c --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf5561b5a4f98495a05489e319bb0dd738c06da1ea4e5a26102d02c2dc6be54b +size 1480575 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6aa92990a709606ab94f998dd937306042117382 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2535ee93601a7918b8bfce14005fc87a9d3bcebbb4b0971083f484b8e6bf71d8 +size 1535847 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..150f5f39cb658be85ed9bbd07f6feef507e504ae --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e356bbb63bbb79e9934d343610bc8c64258a145e769e19279f1394bf799a694 +size 1565153 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fbb469446466f94ab944d0d3d72911f9cde5b2ae --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21895a4eb18f20222f93e9c997afd5ed44a67fc6032cb002d203cb3f84fc63ac +size 1184885 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eeebb9398fc61fc87cc3179278995f3f961d1bf8 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76a7bdc1f11678727d6f4fb49e1c74b004607ee6ee76205af6a182c5bfca6070 +size 1156589 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..750c25a7ef0faecb36dc3b87956ac7b0e8658184 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dc41dd83150fd316589e669dd4ead5d4a0f7cefd969fc964a8154786b5403a0 +size 1164776 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4a30cffc34d96d8609b1badf911f991e09832b0c --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f01db884f36bf3d225fe9af9bfdda114326913abf015553d8f07a4b38a867f5d +size 1156684 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c777e9a73e99966964d11833d4db5eca8412d781 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:593caa1fa9ea606eafa56a443e6ab8b0704d6bbc3a0866c02538c2b1214d14ef +size 1168318 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..358bdf3a5d4b580b1f90550144ac18b881596370 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05651e72bfc1fe624e25ddb4c9d00aa112f0ff67b6a7bbe2a4e4ba3c0ad03806 +size 1167818 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0606b062fd78d1b249bcdf69df93f358bcbd4bf9 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09e9e69382c41ac17586dafc07c5626df519b3bc99dd557ed1209c51ee2a146f +size 1192078 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a59bd85864b9e18331ab1404490272827430d9ca --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d80af6b216e8c15164e950c5bca51c718d88b8343cabba676337bc9f998a21f6 +size 1174056 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d5b1aba80f1e0798ebcdb7c94e99a94d420a2185 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3602d914de1709b10240c32b6e0bb9ee74a8ec5bfda12121d3742a71c7d34246 +size 1182231 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..501646fccaa4fd2ef38994f4e010818412496c9b --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:960ad0adbe0863fa34f9b5075f6b3fb487f931c8f4793fb75e676e9ad15c51a0 +size 1313978 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ed1edff532c4589db55df23a786ed1d0e926672 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a18bdd252b8f5e209e5b888cfe1c153b21be9d07924f54d1b6104d0beb21db7 +size 1160246 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..94f67b98bc59beb3496c6abe90e7ce46c5685f93 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeb6222427bdaa35d11b90fdd007723e3f9b0fc60ce0c7e6a69608e1355f480f +size 1186279 diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a3f92bfc3aa20f8d445be741aca3032adbb1dfbf --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log @@ -0,0 +1,1087 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737248102.837350]: resetting environment +[INFO] [1737248102.844372]: cleared current plan +[INFO] [1737248102.845948]: moving to home +[INFO] [1737248103.848164]: reached home +[INFO] [1737248103.849618]: reset action history +[INFO] [1737248106.559289]: environment reset +Starting episode 0 +[INFO] [1737248106.561172]: resetting environment +[INFO] [1737248106.564688]: cleared current plan +[INFO] [1737248106.565901]: moving to home +[INFO] [1737248107.567694]: reached home +[INFO] [1737248107.569951]: reset action history +[INFO] [1737248110.278143]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737248141.716356]: resetting environment +[INFO] [1737248141.719154]: cleared current plan +[INFO] [1737248141.720004]: moving to home +[INFO] [1737248142.722438]: reached home +[INFO] [1737248142.723889]: reset action history +[INFO] [1737248145.432877]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_0_2025-01-18-19-55-45.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:33195/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2566654] +process[print_text-2]: started with pid [2566655] +process[rosbag_record-3]: started with pid [2566679] +started bagging! +[INFO] [1737248172.404394]: Storing episode... +[print_text-2] killing on exit +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1737248173.154919]: Stored episode 1. +[INFO] [1737248173.155464]: Saving video... +[INFO] [1737248173.346802]: Video saved! +Starting episode 1 +[INFO] [1737248173.364046]: resetting environment +[INFO] [1737248173.371529]: cleared current plan +[INFO] [1737248173.371914]: moving to home +[INFO] [1737248176.874756]: reached home +[INFO] [1737248176.875585]: reset action history +[INFO] [1737248179.589868]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248181.625783]: resetting environment +[INFO] [1737248181.628355]: cleared current plan +[INFO] [1737248181.628522]: moving to home +[INFO] [1737248182.629210]: reached home +[INFO] [1737248182.629634]: reset action history +[INFO] [1737248185.339305]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_1_2025-01-18-19-56-25.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:39513/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2566786] +process[print_text-5]: started with pid [2566787] +process[rosbag_record-6]: started with pid [2566811] +started bagging! +[INFO] [1737248212.314734]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1737248213.098227]: Stored episode 2. +[INFO] [1737248213.099786]: Saving video... +[INFO] [1737248213.320604]: Video saved! +Starting episode 2 +[INFO] [1737248213.354299]: resetting environment +[INFO] [1737248213.364689]: cleared current plan +[INFO] [1737248213.364953]: moving to home +[INFO] [1737248216.969403]: reached home +[INFO] [1737248216.976411]: reset action history +[INFO] [1737248219.760653]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248223.648561]: resetting environment +[INFO] [1737248223.661919]: cleared current plan +[INFO] [1737248223.662975]: moving to home +[INFO] [1737248224.664723]: reached home +[INFO] [1737248224.665213]: reset action history +[INFO] [1737248227.376091]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_2_2025-01-18-19-57-07.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:33565/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2566953] +process[print_text-8]: started with pid [2566954] +process[rosbag_record-9]: started with pid [2566955] +started bagging! +For topic gripper_width: timestamp difference is -104258492 for nearest: 1737248246102267549 - target: 1737248246206526041 at idx 241 +[INFO] [1737248254.707442]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1737248255.485553]: Stored episode 3. +[INFO] [1737248255.487500]: Saving video... +[INFO] [1737248256.012019]: Video saved! +Starting episode 3 +[INFO] [1737248256.056725]: resetting environment +[INFO] [1737248256.069060]: cleared current plan +[INFO] [1737248256.069326]: moving to home +[WARN] [1737248256.034277]: Plan exhausted +[INFO] [1737248259.673774]: reached home +[INFO] [1737248259.681202]: reset action history +[INFO] [1737248262.482557]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248270.081593]: resetting environment +[INFO] [1737248270.102286]: cleared current plan +[INFO] [1737248270.105529]: moving to home +[INFO] [1737248271.108738]: reached home +[INFO] [1737248271.109780]: reset action history +[INFO] [1737248273.910236]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_3_2025-01-18-19-57-53.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:41795/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2567082] +process[print_text-11]: started with pid [2567083] +process[rosbag_record-12]: started with pid [2567107] +started bagging! +For topic gripper_width: timestamp difference is 100519324 for nearest: 1737248293502265597 - target: 1737248293401746273 at idx 249 +For topic gripper_width: timestamp difference is 100847158 for nearest: 1737248293902265939 - target: 1737248293801418781 at idx 249 +[INFO] [1737248300.875909]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1737248301.660888]: Stored episode 4. +[INFO] [1737248301.670636]: Saving video... +[WARN] [1737248302.235839]: Plan exhausted +[WARN] [1737248302.271828]: Plan exhausted +[WARN] [1737248302.311837]: Plan exhausted +[WARN] [1737248302.351835]: Plan exhausted +[INFO] [1737248302.249564]: Video saved! +Starting episode 4 +[INFO] [1737248302.327035]: resetting environment +[INFO] [1737248302.353502]: cleared current plan +[INFO] [1737248302.353716]: moving to home +[INFO] [1737248305.958652]: reached home +[INFO] [1737248305.966820]: reset action history +[INFO] [1737248308.832985]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248311.980460]: resetting environment +[INFO] [1737248312.011646]: cleared current plan +[INFO] [1737248312.017676]: moving to home +[INFO] [1737248313.027435]: reached home +[INFO] [1737248313.032977]: reset action history +[INFO] [1737248315.878925]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_4_2025-01-18-19-58-35.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:35983/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2567236] +process[print_text-14]: started with pid [2567237] +process[rosbag_record-15]: started with pid [2567238] +started bagging! +[INFO] [1737248343.339548]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737248344.134336]: Stored episode 5. +[INFO] [1737248344.137310]: Saving video... +[WARN] [1737248344.672033]: Plan exhausted +[INFO] [1737248344.621695]: Video saved! +Starting episode 5 +[INFO] [1737248344.679258]: resetting environment +[INFO] [1737248344.695760]: cleared current plan +[INFO] [1737248344.702244]: moving to home +[INFO] [1737248348.313158]: reached home +[INFO] [1737248348.317041]: reset action history +[INFO] [1737248351.244774]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248362.804807]: resetting environment +[INFO] [1737248362.836175]: cleared current plan +[INFO] [1737248362.844832]: moving to home +[INFO] [1737248363.867538]: reached home +[INFO] [1737248363.869356]: reset action history +[INFO] [1737248366.724348]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_5_2025-01-18-19-59-26.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:42821/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2567382] +process[print_text-17]: started with pid [2567383] +process[rosbag_record-18]: started with pid [2567384] +started bagging! +For topic gripper_width: timestamp difference is 100444401 for nearest: 1737248377302266205 - target: 1737248377201821804 at idx 131 +[INFO] [1737248393.885206]: Storing episode... +[print_text-17] killing on exit +[rosbag_record-18] killing on exit +[pub_text-16] killing on exit +[INFO] [1737248394.713489]: Stored episode 6. +[INFO] [1737248394.726552]: Saving video... +[INFO] [1737248395.032738]: Video saved! +Starting episode 6 +[INFO] [1737248395.123948]: resetting environment +[INFO] [1737248395.167063]: cleared current plan +[INFO] [1737248395.174253]: moving to home +[INFO] [1737248398.791849]: reached home +[INFO] [1737248398.805246]: reset action history +[INFO] [1737248401.707480]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248404.826130]: resetting environment +[INFO] [1737248404.865891]: cleared current plan +[INFO] [1737248404.867944]: moving to home +[INFO] [1737248405.880600]: reached home +[INFO] [1737248405.888882]: reset action history +[INFO] [1737248408.726461]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_6_2025-01-18-20-00-08.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:39991/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2567526] +process[print_text-20]: started with pid [2567527] +process[rosbag_record-21]: started with pid [2567528] +started bagging! +For topic gripper_width: timestamp difference is 101472193 for nearest: 1737248419568883234 - target: 1737248419467411041 at idx 132 +For topic gripper_width: timestamp difference is 107653105 for nearest: 1737248422568933689 - target: 1737248422461280584 at idx 177 +For topic gripper_width: timestamp difference is 107793193 for nearest: 1737248426368897300 - target: 1737248426261104107 at idx 231 +For topic gripper_width: timestamp difference is 108306081 for nearest: 1737248426968932779 - target: 1737248426860626698 at idx 237 +For topic gripper_width: timestamp difference is 108494106 for nearest: 1737248427168961826 - target: 1737248427060467720 at idx 239 +For topic gripper_width: timestamp difference is 111143644 for nearest: 1737248430369026716 - target: 1737248430257883072 at idx 249 +For topic gripper_width: timestamp difference is 115215576 for nearest: 1737248435568932808 - target: 1737248435453717232 at idx 249 +[INFO] [1737248435.934048]: Storing episode... +[rosbag_record-21] killing on exit[print_text-20] killing on exit +[pub_text-19] killing on exit +[WARN] [1737248437.073754]: Plan exhausted +[WARN] [1737248437.111894]: Plan exhausted +[WARN] [1737248437.151938]: Plan exhausted +[WARN] [1737248437.197056]: Plan exhausted +[WARN] [1737248437.232726]: Plan exhausted +[WARN] [1737248437.274002]: Plan exhausted +[WARN] [1737248437.313686]: Plan exhausted +[WARN] [1737248437.354721]: Plan exhausted +[WARN] [1737248437.392658]: Plan exhausted +[WARN] [1737248437.435493]: Plan exhausted +[WARN] [1737248437.473216]: Plan exhausted +[WARN] [1737248437.517296]: Plan exhausted +[WARN] [1737248437.551825]: Plan exhausted +[WARN] [1737248437.598107]: Plan exhausted +[WARN] [1737248437.636661]: Plan exhausted +[INFO] [1737248436.790901]: Stored episode 7. +[INFO] [1737248436.805612]: Saving video... +[INFO] [1737248437.529316]: Video saved! +Starting episode 7 +[INFO] [1737248437.611361]: resetting environment +[INFO] [1737248437.649384]: cleared current plan +[INFO] [1737248437.667418]: moving to home +[INFO] [1737248441.306600]: reached home +[INFO] [1737248441.329880]: reset action history +[INFO] [1737248444.302880]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248448.271413]: resetting environment +[INFO] [1737248448.316030]: cleared current plan +[INFO] [1737248448.326764]: moving to home +[INFO] [1737248449.335158]: reached home +[INFO] [1737248449.345461]: reset action history +[INFO] [1737248452.294938]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_7_2025-01-18-20-00-52.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:34161/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2567671] +process[print_text-23]: started with pid [2567672] +process[rosbag_record-24]: started with pid [2567673] +started bagging! +For topic gripper_width: timestamp difference is -115479411 for nearest: 1737248479902224122 - target: 1737248480017703533 at idx 249 +[INFO] [1737248482.886175]: Storing episode... +[rosbag_record-24] killing on exit +[print_text-23] killing on exit +[pub_text-22] killing on exit +[INFO] [1737248483.688649]: Stored episode 8. +[INFO] [1737248483.707079]: Saving video... +[INFO] [1737248484.130374]: Video saved! +Starting episode 8 +[INFO] [1737248484.225124]: resetting environment +[INFO] [1737248484.254501]: cleared current plan +[INFO] [1737248484.254786]: moving to home +[WARN] [1737248484.232567]: Plan exhausted +[INFO] [1737248487.860257]: reached home +[INFO] [1737248487.872402]: reset action history +[INFO] [1737248490.805315]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248494.738464]: resetting environment +[INFO] [1737248494.796167]: cleared current plan +[INFO] [1737248494.798590]: moving to home +[INFO] [1737248495.822706]: reached home +[INFO] [1737248495.844803]: reset action history +[INFO] [1737248498.831259]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_8_2025-01-18-20-01-38.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:41797/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2567781] +process[print_text-26]: started with pid [2567782] +process[rosbag_record-27]: started with pid [2567783] +started bagging! +[INFO] [1737248525.805970]: Storing episode... +[pub_text-25] killing on exit +[rosbag_record-27] killing on exit +[print_text-26] killing on exit +[WARN] [1737248526.952894]: Plan exhausted +[WARN] [1737248526.993238]: Plan exhausted +[WARN] [1737248527.031969]: Plan exhausted +[WARN] [1737248527.073837]: Plan exhausted +[WARN] [1737248527.113294]: Plan exhausted +[WARN] [1737248527.153232]: Plan exhausted +[WARN] [1737248527.194610]: Plan exhausted +[WARN] [1737248527.234159]: Plan exhausted +[WARN] [1737248527.273051]: Plan exhausted +[INFO] [1737248526.659164]: Stored episode 9. +[INFO] [1737248526.671568]: Saving video... +[INFO] [1737248527.129174]: Video saved! +Starting episode 9 +[INFO] [1737248527.243123]: resetting environment +[INFO] [1737248527.291997]: cleared current plan +[INFO] [1737248527.307996]: moving to home +[INFO] [1737248530.927663]: reached home +[INFO] [1737248530.933549]: reset action history +[INFO] [1737248533.991174]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248537.603884]: resetting environment +[INFO] [1737248537.655321]: cleared current plan +[INFO] [1737248537.663436]: moving to home +[INFO] [1737248538.678607]: reached home +[INFO] [1737248538.718071]: reset action history +[INFO] [1737248541.771253]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_9_2025-01-18-20-02-21.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:41945/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2567951] +process[print_text-29]: started with pid [2567952] +process[rosbag_record-30]: started with pid [2567953] +started bagging! +[INFO] [1737248569.065231]: Storing episode... +[print_text-29] killing on exit +[rosbag_record-30] killing on exit +[pub_text-28] killing on exit +[INFO] [1737248569.900649]: Stored episode 10. +[INFO] [1737248569.925997]: Saving video... +[INFO] [1737248570.298412]: Video saved! +Starting episode 10 +[INFO] [1737248570.425471]: resetting environment +[INFO] [1737248570.473392]: cleared current plan +[INFO] [1737248570.473747]: moving to home +[WARN] [1737248570.392138]: Plan exhausted +[WARN] [1737248570.433606]: Plan exhausted +[WARN] [1737248570.471811]: Plan exhausted +[INFO] [1737248574.085462]: reached home +[INFO] [1737248574.099539]: reset action history +[INFO] [1737248577.111243]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248580.760771]: resetting environment +[INFO] [1737248580.824421]: cleared current plan +[INFO] [1737248580.854817]: moving to home +[INFO] [1737248581.879827]: reached home +[INFO] [1737248581.891926]: reset action history +[INFO] [1737248584.881614]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_10_2025-01-18-20-03-04.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:37017/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2568098] +process[print_text-32]: started with pid [2568099] +process[rosbag_record-33]: started with pid [2568100] +started bagging! +[INFO] [1737248616.902532]: Storing episode... +[rosbag_record-33] killing on exit +[print_text-32] killing on exit +[pub_text-31] killing on exit +[INFO] [1737248617.730808]: Stored episode 11. +[INFO] [1737248617.746796]: Saving video... +[INFO] [1737248617.991861]: Video saved! +Starting episode 11 +[INFO] [1737248618.138119]: resetting environment +[INFO] [1737248618.207731]: cleared current plan +[INFO] [1737248618.216843]: moving to home +[WARN] [1737248618.073920]: Plan exhausted +[WARN] [1737248618.113470]: Plan exhausted +[WARN] [1737248618.167545]: Plan exhausted +[WARN] [1737248618.198988]: Plan exhausted +[INFO] [1737248621.827960]: reached home +[INFO] [1737248621.831947]: reset action history +[INFO] [1737248624.892116]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248629.369781]: resetting environment +[INFO] [1737248629.385006]: cleared current plan +[INFO] [1737248629.385211]: moving to home +[INFO] [1737248630.389079]: reached home +[INFO] [1737248630.416536]: reset action history +[INFO] [1737248633.517942]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_11_2025-01-18-20-03-53.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:35799/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2568243] +process[print_text-35]: started with pid [2568244] +process[rosbag_record-36]: started with pid [2568245] +started bagging! +[INFO] [1737248666.789807]: Storing episode... +[rosbag_record-36] killing on exit +[pub_text-34] killing on exit +[print_text-35] killing on exit +[INFO] [1737248667.591953]: Stored episode 12. +[INFO] [1737248667.612942]: Saving video... +[INFO] [1737248668.252047]: Video saved! +Starting episode 12 +[INFO] [1737248668.420029]: resetting environment +[INFO] [1737248668.489548]: cleared current plan +[INFO] [1737248668.496870]: moving to home +[WARN] [1737248667.952697]: Plan exhausted +[WARN] [1737248667.992705]: Plan exhausted +[WARN] [1737248668.041374]: Plan exhausted +[WARN] [1737248668.076601]: Plan exhausted +[WARN] [1737248668.112745]: Plan exhausted +[WARN] [1737248668.152512]: Plan exhausted +[WARN] [1737248668.193833]: Plan exhausted +[WARN] [1737248668.233032]: Plan exhausted +[WARN] [1737248668.271821]: Plan exhausted +[WARN] [1737248668.317798]: Plan exhausted +[WARN] [1737248668.357302]: Plan exhausted +[WARN] [1737248668.395727]: Plan exhausted +[WARN] [1737248668.442039]: Plan exhausted +[WARN] [1737248668.472420]: Plan exhausted +[INFO] [1737248672.120394]: reached home +[INFO] [1737248672.125361]: reset action history +[INFO] [1737248675.355398]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248678.454654]: resetting environment +[INFO] [1737248678.533297]: cleared current plan +[INFO] [1737248678.549223]: moving to home +[INFO] [1737248679.576642]: reached home +[INFO] [1737248679.585583]: reset action history +[INFO] [1737248682.715703]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_12_2025-01-18-20-04-42.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:34841/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2568352] +process[print_text-38]: started with pid [2568353] +process[rosbag_record-39]: started with pid [2568354] +started bagging! +[INFO] [1737248709.725747]: Storing episode... +[pub_text-37] killing on exit +[rosbag_record-39] killing on exit +[print_text-38] killing on exit +[INFO] [1737248710.519421]: Stored episode 13. +[INFO] [1737248710.544843]: Saving video... +[WARN] [1737248711.079226]: Plan exhausted +[WARN] [1737248711.115539]: Plan exhausted +[WARN] [1737248711.152159]: Plan exhausted +[WARN] [1737248711.198286]: Plan exhausted +[WARN] [1737248711.236874]: Plan exhausted +[WARN] [1737248711.271816]: Plan exhausted +[WARN] [1737248711.317620]: Plan exhausted +[WARN] [1737248711.354653]: Plan exhausted +[INFO] [1737248711.179618]: Video saved! +Starting episode 13 +[INFO] [1737248711.310899]: resetting environment +[INFO] [1737248711.368953]: cleared current plan +[INFO] [1737248711.382805]: moving to home +[INFO] [1737248714.993449]: reached home +[INFO] [1737248715.039355]: reset action history +proceeding to start episode! +[INFO] [1737248718.280782]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248756.593266]: resetting environment +[INFO] [1737248756.659100]: cleared current plan +[INFO] [1737248756.683735]: moving to home +[INFO] [1737248757.709341]: reached home +[INFO] [1737248757.729651]: reset action history +[INFO] [1737248760.846509]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_13_2025-01-18-20-06-00.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:40889/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2568523] +process[print_text-41]: started with pid [2568524] +process[rosbag_record-42]: started with pid [2568525] +started bagging! +[INFO] [1737248796.190780]: Storing episode... +[rosbag_record-42] killing on exit +[print_text-41] killing on exit +[pub_text-40] killing on exit +[WARN] [1737248797.551835]: Plan exhausted +[WARN] [1737248797.591830]: Plan exhausted +[INFO] [1737248797.041810]: Stored episode 14. +[INFO] [1737248797.090444]: Saving video... +[INFO] [1737248797.434047]: Video saved! +Starting episode 14 +[INFO] [1737248797.561544]: resetting environment +[INFO] [1737248797.597090]: cleared current plan +[INFO] [1737248797.606362]: moving to home +[INFO] [1737248801.223988]: reached home +[INFO] [1737248801.250697]: reset action history +[INFO] [1737248804.449222]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248809.472008]: resetting environment +[INFO] [1737248809.541242]: cleared current plan +[INFO] [1737248809.549067]: moving to home +[INFO] [1737248810.578781]: reached home +[INFO] [1737248810.595210]: reset action history +[INFO] [1737248813.706625]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_14_2025-01-18-20-06-53.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:34337/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2568668] +process[print_text-44]: started with pid [2568669] +process[rosbag_record-45]: started with pid [2568670] +started bagging! +For topic gripper_width: timestamp difference is -130978198 for nearest: 1737248832102262406 - target: 1737248832233240604 at idx 249 +[INFO] [1737248849.492150]: Storing episode... +[print_text-44] killing on exit +[pub_text-43] killing on exit +[rosbag_record-45] killing on exit +[INFO] [1737248850.302564]: Stored episode 15. +[INFO] [1737248850.307817]: Saving video... +[WARN] [1737248850.833070]: Plan exhausted +[WARN] [1737248850.872517]: Plan exhausted +[WARN] [1737248850.915877]: Plan exhausted +[WARN] [1737248850.960838]: Plan exhausted +[WARN] [1737248850.999633]: Plan exhausted +[WARN] [1737248851.037053]: Plan exhausted +[INFO] [1737248850.907256]: Video saved! +Starting episode 15 +[INFO] [1737248851.021312]: resetting environment +[INFO] [1737248851.048430]: cleared current plan +[INFO] [1737248851.053077]: moving to home +[INFO] [1737248854.667685]: reached home +[INFO] [1737248854.678768]: reset action history +[INFO] [1737248857.956640]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248860.293869]: resetting environment +[INFO] [1737248860.325620]: cleared current plan +[INFO] [1737248860.331381]: moving to home +[INFO] [1737248861.353426]: reached home +[INFO] [1737248861.368341]: reset action history +[INFO] [1737248864.611240]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_15_2025-01-18-20-07-44.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:42561/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2568804] +process[print_text-47]: started with pid [2568805] +process[rosbag_record-48]: started with pid [2568806] +started bagging! +For topic gripper_width: timestamp difference is 107138963 for nearest: 1737248879602265210 - target: 1737248879495126247 at idx 230 +For topic gripper_width: timestamp difference is 107750281 for nearest: 1737248880402262314 - target: 1737248880294512033 at idx 239 +For topic gripper_width: timestamp difference is -218893350 for nearest: 1737248888302261292 - target: 1737248888521154642 at idx 249 +[INFO] [1737248901.184509]: Storing episode... +[pub_text-46] killing on exit +[rosbag_record-48] killing on exit[print_text-47] killing on exit +[INFO] [1737248902.065632]: Stored episode 16. +[INFO] [1737248902.091790]: Saving video... +[INFO] [1737248902.420961]: Video saved! +Starting episode 16 +[INFO] [1737248902.595473]: resetting environment +[INFO] [1737248902.629155]: cleared current plan +[INFO] [1737248902.641484]: moving to home +[WARN] [1737248902.551843]: Plan exhausted +[WARN] [1737248902.606426]: Plan exhausted +[INFO] [1737248906.273025]: reached home +[INFO] [1737248906.296618]: reset action history +[INFO] [1737248909.642111]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248912.420275]: resetting environment +[INFO] [1737248912.563709]: cleared current plan +[INFO] [1737248912.594197]: moving to home +[INFO] [1737248913.648546]: reached home +[INFO] [1737248913.669116]: reset action history +[INFO] [1737248916.891037]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_16_2025-01-18-20-08-36.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:43565/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2568939] +process[print_text-50]: started with pid [2568940] +process[rosbag_record-51]: started with pid [2568941] +started bagging! +For topic gripper_width: timestamp difference is -101428972 for nearest: 1737248951302261366 - target: 1737248951403690338 at idx 249 +For topic gripper_width: timestamp difference is -100170474 for nearest: 1737248952902220149 - target: 1737248953002390623 at idx 249 +[INFO] [1737248954.054966]: Storing episode... +[pub_text-49] killing on exit[print_text-50] killing on exit +[rosbag_record-51] killing on exit +[INFO] [1737248954.925305]: Stored episode 17. +[WARN] [1737248955.447837]: Plan exhausted +[WARN] [1737248955.474548]: Plan exhausted +[WARN] [1737248955.516996]: Plan exhausted +[WARN] [1737248955.554084]: Plan exhausted +[WARN] [1737248955.593688]: Plan exhausted +[WARN] [1737248955.632190]: Plan exhausted +[WARN] [1737248955.690474]: Plan exhausted +[WARN] [1737248955.722887]: Plan exhausted +[WARN] [1737248955.783295]: Plan exhausted +[WARN] [1737248955.804689]: Plan exhausted +[WARN] [1737248955.865985]: Plan exhausted +[WARN] [1737248955.890698]: Plan exhausted +[INFO] [1737248954.952783]: Saving video... +[INFO] [1737248955.653418]: Video saved! +Starting episode 17 +[INFO] [1737248955.834234]: resetting environment +[INFO] [1737248955.896503]: cleared current plan +[INFO] [1737248955.897895]: moving to home +[INFO] [1737248959.523703]: reached home +[INFO] [1737248959.569009]: reset action history +[INFO] [1737248962.840485]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737248967.140472]: resetting environment +[INFO] [1737248967.263711]: cleared current plan +[INFO] [1737248967.274586]: moving to home +[INFO] [1737248968.338595]: reached home +[INFO] [1737248968.372744]: reset action history +[INFO] [1737248971.669533]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_17_2025-01-18-20-09-31.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:39721/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2569071] +process[print_text-53]: started with pid [2569072] +process[rosbag_record-54]: started with pid [2569073] +started bagging! +For topic gripper_width: timestamp difference is -206586378 for nearest: 1737248986102259380 - target: 1737248986308845758 at idx 199 +For topic gripper_width: timestamp difference is -406416195 for nearest: 1737248986102259380 - target: 1737248986508675575 at idx 199 +For topic gripper_width: timestamp difference is 101322146 for nearest: 1737248996202262850 - target: 1737248996100940704 at idx 249 +[INFO] [1737248998.987518]: Storing episode... +[rosbag_record-54] killing on exit +[pub_text-52] killing on exit +[print_text-53] killing on exit +[INFO] [1737248999.867389]: Stored episode 18. +[INFO] [1737248999.925553]: Saving video... +[INFO] [1737249000.186351]: Video saved! +Starting episode 18 +[INFO] [1737249000.378355]: resetting environment +[INFO] [1737249000.447515]: cleared current plan +[INFO] [1737249000.460443]: moving to home +[WARN] [1737249000.328606]: Plan exhausted +[WARN] [1737249000.363211]: Plan exhausted +[WARN] [1737249000.393557]: Plan exhausted +[WARN] [1737249000.431799]: Plan exhausted +[INFO] [1737249004.131557]: reached home +[INFO] [1737249004.170863]: reset action history +[INFO] [1737249007.647236]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249009.464525]: resetting environment +[INFO] [1737249009.561933]: cleared current plan +[INFO] [1737249009.571107]: moving to home +[INFO] [1737249010.617434]: reached home +[INFO] [1737249010.637270]: reset action history +[INFO] [1737249013.908531]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_18_2025-01-18-20-10-13.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:33837/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2569180] +process[print_text-56]: started with pid [2569204] +process[rosbag_record-57]: started with pid [2569205] +started bagging! +For topic gripper_width: timestamp difference is -246739044 for nearest: 1737249018802261696 - target: 1737249019049000740 at idx 73 +For topic gripper_width: timestamp difference is 103154007 for nearest: 1737249039735594574 - target: 1737249039632440567 at idx 249 +[INFO] [1737249040.915237]: Storing episode... +[rosbag_record-57] killing on exit +[pub_text-55] killing on exit[print_text-56] killing on exit +[WARN] [1737249042.239793]: Plan exhausted +[INFO] [1737249041.778384]: Stored episode 19. +[INFO] [1737249041.818560]: Saving video... +[INFO] [1737249042.466493]: Video saved! +Starting episode 19 +[INFO] [1737249042.689922]: resetting environment +[INFO] [1737249042.812840]: cleared current plan +[INFO] [1737249042.821006]: moving to home +[WARN] [1737249042.294814]: Plan exhausted +[WARN] [1737249042.319669]: Plan exhausted +[WARN] [1737249042.353636]: Plan exhausted +[WARN] [1737249042.396114]: Plan exhausted +[WARN] [1737249042.439584]: Plan exhausted +[WARN] [1737249042.483474]: Plan exhausted +[WARN] [1737249042.515794]: Plan exhausted +[WARN] [1737249042.553056]: Plan exhausted +[WARN] [1737249042.606110]: Plan exhausted +[WARN] [1737249042.648617]: Plan exhausted +[WARN] [1737249042.732600]: Plan exhausted +[WARN] [1737249042.781111]: Plan exhausted +[WARN] [1737249042.786681]: Plan exhausted +[WARN] [1737249042.791836]: Plan exhausted +[INFO] [1737249046.565644]: reached home +[INFO] [1737249046.582916]: reset action history +[INFO] [1737249049.887859]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249056.582915]: resetting environment +[INFO] [1737249056.650466]: cleared current plan +[INFO] [1737249056.674026]: moving to home +[INFO] [1737249057.749198]: reached home +[INFO] [1737249057.829394]: reset action history +[INFO] [1737249061.073901]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/episode_rosbags/episode_19_2025-01-18-20-11-01.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2566358.log +started roslaunch server http://158.130.50.37:35637/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2569326] +process[print_text-59]: started with pid [2569327] +process[rosbag_record-60]: started with pid [2569351] +started bagging! +[INFO] [1737249088.102337]: Storing episode... +[pub_text-58] killing on exit +[rosbag_record-60] killing on exit +[print_text-59] killing on exit +[INFO] [1737249088.964631]: Stored episode 20. +[INFO] [1737249088.986320]: Saving video... +[INFO] [1737249089.241375]: Video saved! +[INFO] [1737249089.349108]: resetting environment +[INFO] [1737249089.392459]: cleared current plan +[INFO] [1737249089.410800]: moving to home +[INFO] [1737249093.035673]: reached home +[INFO] [1737249093.074838]: reset action history +[INFO] [1737249096.476134]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/requirements.txt b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..a5389e74874ef2add6aa3eadc61bc8ee3ea4ebde --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,119 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..03918f5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -132,15 +132,25 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++wandb_run_id: '3465_0' ++# wandb_run_id: '3465_1' # seed 1 ++# wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-# wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +@@ -191,7 +201,7 @@ wandb_run_id: '1017_0' # dataset shuffle seed 1 + true_action_history: false + wandb_notes: null + +-checkpoint_epoch: 12000 # for depth + all contact ++checkpoint_epoch: 7500 # for depth + all contact + load_residual_weight: false + + checkpoint_root_dir: '/home/${oc.env:USER}/fish_leon/FISH' +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..9509617 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 7500 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-metadata.json b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..eead52c9944cbab9fd8f6d19cd1cccf3fe8ac0bf --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-19T00:54:58.059031", + "startedAt": "2025-01-19T00:54:57.673346", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3559.8128749999996, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4489.241, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 579.0002593994141 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5b34919e4e3b7af297dc716f9cdf378996438902 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "dd640c135955b3d54fec7ae6fb41c2edef3b17fda1044801e3101daccb7fcae9", "size": 1185117, "path": "media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4"}, "global_step": 2655, "_timestamp": 1737249100.993931, "_runtime": 1003.3050601482391, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "f01db884f36bf3d225fe9af9bfdda114326913abf015553d8f07a4b38a867f5d", "size": 1156684, "path": "media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4"}, "eval/num_success": 21.0, "episode": 20.0, "eval/success_rate": 1.0499999523162842, "eval/success_rate_lb_0.01": 1.0, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 1.0, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 1.0, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "593caa1fa9ea606eafa56a443e6ab8b0704d6bbc3a0866c02538c2b1214d14ef", "size": 1168318, "path": "media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "05651e72bfc1fe624e25ddb4c9d00aa112f0ff67b6a7bbe2a4e4ba3c0ad03806", "size": 1167818, "path": "media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "09e9e69382c41ac17586dafc07c5626df519b3bc99dd557ed1209c51ee2a146f", "size": 1192078, "path": "media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "d80af6b216e8c15164e950c5bca51c718d88b8343cabba676337bc9f998a21f6", "size": 1174056, "path": "media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "3602d914de1709b10240c32b6e0bb9ee74a8ec5bfda12121d3742a71c7d34246", "size": 1182231, "path": "media/videos/eval/6_eval_6_3602d914de1709b10240.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "960ad0adbe0863fa34f9b5075f6b3fb487f931c8f4793fb75e676e9ad15c51a0", "size": 1313978, "path": "media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "4a18bdd252b8f5e209e5b888cfe1c153b21be9d07924f54d1b6104d0beb21db7", "size": 1160246, "path": "media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "eeb6222427bdaa35d11b90fdd007723e3f9b0fc60ce0c7e6a69608e1355f480f", "size": 1186279, "path": "media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "708e5d86baaf5c876d21100f22ad7b92231cbe37549a12b676e35762ac00de0b", "size": 1369725, "path": "media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "5e48fceb469367c5942f582757e64e07e7cccfbdf6b03d7970d1ccbe00037983", "size": 1401639, "path": "media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "7a56ddf9bcd287e59382b84cd2c72971de6aaa276cbfe72b32f05001e3a2a734", "size": 1170389, "path": "media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "ce205101458cc00950f123e1c7e065c5f0df37ab903d9ab26f3e942715d9d157", "size": 1488237, "path": "media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "bf5561b5a4f98495a05489e319bb0dd738c06da1ea4e5a26102d02c2dc6be54b", "size": 1480575, "path": "media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "2535ee93601a7918b8bfce14005fc87a9d3bcebbb4b0971083f484b8e6bf71d8", "size": 1535847, "path": "media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "7e356bbb63bbb79e9934d343610bc8c64258a145e769e19279f1394bf799a694", "size": 1565153, "path": "media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "21895a4eb18f20222f93e9c997afd5ed44a67fc6032cb002d203cb3f84fc63ac", "size": 1184885, "path": "media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "76a7bdc1f11678727d6f4fb49e1c74b004607ee6ee76205af6a182c5bfca6070", "size": 1156589, "path": "media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "0dc41dd83150fd316589e669dd4ead5d4a0f7cefd969fc964a8154786b5403a0", "size": 1164776, "path": "media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "6cabaaf3e79ec48751e3e6077ecdff7619c089ba5f8039faf091902164803f26", "size": 263, "artifact_path": "wandb-client-artifact://6inc00wvi3i5tex1w5ed6t1nfo2p5j6o3cuqt7wmwyznsoa0e4kt0k9kk5u3ynzntrtkd7xtcqucep390tegzemak61f9cynnitm9s3erys0zc1y6kbewisfc1e1npzz/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://jzurowz19qpj6g2pfzn71wy6wsau9tyhlspfgnkc3d2pgfjvgshz3cegff6xxlvvf21sicyohogpdxnluzbu69pl0mkci6y8822hdpqinam2viy1d6crw240irdbxm40:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 21, "total_num_episodes": 20, "total_success_rate": 1.05, "total_success_rate_lb_0.01": 1, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 1, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 1, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "109ab103ab9a05e5d5754d51851601a1d9864986dcdbb581bd686576a70e6a84", "size": 370, "artifact_path": "wandb-client-artifact://qv4w9n31b5qztyjmejix9u0om2vexyxefzera3np50sp5jdnx6fk6ufc6qp4f6o7qzcx661o74twz5ijthw8hc2ww5ojkuflyc1s3b087vfb97qt5xpt6jmeh49mhyqe/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://8kueupcgmkzz32qn7aogqkbtbkkxol9x93z1zwvhugm87bwwyaruts3qqbhcbcssyu6aua6npbl7vg0ot8ta4nkem4day0sdqr7aqpmj5hq1sca8g71cz2xf26k2uloe:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1035}, "eval/total_success_rate": 1.0499999523162842, "eval/total_success_rate_lb_0.01": 1.0, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 1.0, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 1.0, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug-internal.log b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f0388f5cf579ca3fdd992ee4d39acb0128c6891d --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug-internal.log @@ -0,0 +1,2199 @@ +2025-01-18 19:54:57,688 INFO StreamThr :2566524 [internal.py:wandb_internal():85] W&B internal server running at pid: 2566524, started at: 2025-01-18 19:54:57.688322 +2025-01-18 19:54:57,689 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status +2025-01-18 19:54:57,701 INFO WriterThread:2566524 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/run-mo0b6mh9.wandb +2025-01-18 19:54:57,701 DEBUG SenderThread:2566524 [sender.py:send():379] send: header +2025-01-18 19:54:57,710 DEBUG SenderThread:2566524 [sender.py:send():379] send: run +2025-01-18 19:54:57,948 INFO SenderThread:2566524 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files +2025-01-18 19:54:57,949 INFO SenderThread:2566524 [sender.py:_start_run_threads():1188] run started: mo0b6mh9 with start time 1737248097.688871 +2025-01-18 19:54:57,963 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 19:54:57,963 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: check_version +2025-01-18 19:54:58,036 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 19:54:58,044 DEBUG HandlerThread:2566524 [system_info.py:__init__():26] System info init +2025-01-18 19:54:58,044 DEBUG HandlerThread:2566524 [system_info.py:__init__():41] System info init done +2025-01-18 19:54:58,045 INFO HandlerThread:2566524 [system_monitor.py:start():194] Starting system monitor +2025-01-18 19:54:58,045 INFO SystemMonitor:2566524 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 19:54:58,045 INFO HandlerThread:2566524 [system_monitor.py:probe():214] Collecting system info +2025-01-18 19:54:58,045 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 19:54:58,047 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started disk monitoring +2025-01-18 19:54:58,047 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 19:54:58,048 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started memory monitoring +2025-01-18 19:54:58,048 INFO SystemMonitor:2566524 [interfaces.py:start():188] Started network monitoring +2025-01-18 19:54:58,058 DEBUG HandlerThread:2566524 [system_info.py:probe():152] Probing system +2025-01-18 19:54:58,060 DEBUG HandlerThread:2566524 [system_info.py:_probe_git():137] Probing git +2025-01-18 19:54:58,064 DEBUG HandlerThread:2566524 [system_info.py:_probe_git():145] Probing git done +2025-01-18 19:54:58,065 DEBUG HandlerThread:2566524 [system_info.py:probe():200] Probing system done +2025-01-18 19:54:58,065 DEBUG HandlerThread:2566524 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T00:54:58.059031', 'startedAt': '2025-01-19T00:54:57.673346', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3559.8128749999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 4489.241, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 579.0002593994141}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 19:54:58,065 INFO HandlerThread:2566524 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 19:54:58,065 INFO HandlerThread:2566524 [system_monitor.py:probe():227] Publishing system info +2025-01-18 19:54:58,065 DEBUG HandlerThread:2566524 [system_info.py:_save_code():44] Saving code +2025-01-18 19:54:58,070 DEBUG HandlerThread:2566524 [system_info.py:_save_code():67] Saving code done +2025-01-18 19:54:58,070 DEBUG HandlerThread:2566524 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 19:54:58,168 DEBUG HandlerThread:2566524 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 19:54:58,169 INFO HandlerThread:2566524 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 19:54:58,172 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:54:58,172 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 19:54:58,172 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 19:54:58,173 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 19:54:58,173 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 19:54:58,402 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 19:54:58,403 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: python_packages +2025-01-18 19:54:58,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:54:58,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:54:58,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:54:58,435 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/vx0a3pt4-diff.patch +2025-01-18 19:54:58,438 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ibb2j8co-wandb-metadata.json +2025-01-18 19:54:58,445 INFO HandlerThread:2566524 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/tb" + save: true +} +control { + relay_id: "70c5eacc984a4afebe7199c0bf475f13" +} +_info { + stream_id: "mo0b6mh9" +} + +2025-01-18 19:54:58,445 DEBUG HandlerThread:2566524 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 19:54:58,452 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/x1yqpgel-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:54:58,483 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/wt9sjipv-code/FISH/eval_robot.py +2025-01-18 19:54:58,498 DEBUG SenderThread:2566524 [sender.py:send():379] send: telemetry +2025-01-18 19:54:58,498 DEBUG SenderThread:2566524 [sender.py:send():379] send: tbrecord +2025-01-18 19:54:58,499 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:54:58,499 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 with policy live +2025-01-18 19:54:58,685 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/2ud0oelk-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:54:58,950 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:54:58,950 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/diff.patch +2025-01-18 19:54:58,950 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-metadata.json +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code/FISH/eval_robot.py +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/requirements.txt +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code/FISH +2025-01-18 19:54:58,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code +2025-01-18 19:54:59,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:00,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:00,949 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:01,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:02,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:02,836 DEBUG SenderThread:2566524 [sender.py:send():379] send: config +2025-01-18 19:55:02,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:03,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:04,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:04,951 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:05,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:06,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:07,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:08,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:08,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:08,952 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:09,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:10,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:11,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:12,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:12,953 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:13,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:55:13,403 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:55:13,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:13,497 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:14,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:15,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:16,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:17,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:18,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:19,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:19,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:20,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:21,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:22,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:23,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:24,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:25,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:25,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:26,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:27,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:28,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:55:28,403 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:55:28,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:29,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:30,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:30,558 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:30,958 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml +2025-01-18 19:55:31,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:32,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:33,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:34,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:35,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:36,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:36,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:37,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:38,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:39,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:40,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:41,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:41,716 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:42,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:42,961 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:43,403 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:55:43,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:55:43,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:44,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:44,962 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:45,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:46,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:46,963 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:55:47,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:47,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:48,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:49,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:50,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:51,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:52,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:53,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:53,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:54,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:55,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:56,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:57,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:58,049 DEBUG SystemMonitor:2566524 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 19:55:58,049 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:55:58,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:55:58,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:55:58,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:55:58,463 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:55:59,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:00,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:01,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:02,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:03,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:04,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:04,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:05,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:06,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:07,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:08,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:09,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:09,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:10,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:11,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:12,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:13,362 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:56:13,363 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 with policy now +2025-01-18 19:56:13,363 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:56:13,364 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:56:13,365 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:56:13,365 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:56:13,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:56:13,405 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:56:13,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:13,622 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/c33jqz39-media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos +2025-01-18 19:56:13,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media +2025-01-18 19:56:14,242 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/5q0i49hy-media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 +2025-01-18 19:56:14,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:14,506 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:14,970 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:15,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:16,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:17,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:18,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:18,972 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:19,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:19,590 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:20,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:20,972 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:21,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:21,973 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:56:22,218 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/vqv8y13n-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:56:22,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:22,973 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:23,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:24,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:24,974 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:25,339 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:25,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:26,416 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:26,975 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:27,417 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:28,081 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:56:28,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:56:28,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:56:28,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:29,417 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:30,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:30,526 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:31,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:32,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:33,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:34,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:35,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:36,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:36,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:37,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:38,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:39,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:40,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:41,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:42,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:42,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:43,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:56:43,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:56:43,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:44,419 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:45,419 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:46,419 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:47,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:47,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:48,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:49,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:50,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:51,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:52,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:53,098 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:53,351 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:56:53,352 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 with policy now +2025-01-18 19:56:53,354 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:56:53,355 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:56:53,355 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:56:53,356 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:56:53,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:53,621 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/cj77boio-media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 +2025-01-18 19:56:53,984 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:56:53,984 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 +2025-01-18 19:56:53,984 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:56:54,253 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/bkkpb6kx-media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 +2025-01-18 19:56:54,423 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:54,984 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:55,425 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:56,423 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:57,440 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:58,050 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:56:58,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:56:58,409 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:56:58,453 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:56:58,565 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:56:58,985 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:56:59,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:00,431 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:00,986 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:01,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:02,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:03,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:03,634 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:03,860 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:57:03,860 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:57:03,861 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:57:03,987 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:57:03,987 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:57:04,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:04,987 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:05,431 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:06,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:06,988 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:07,438 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:08,432 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:08,989 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:09,409 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:09,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:10,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:11,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:12,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:13,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:57:13,405 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:57:13,449 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:14,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:14,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:15,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:16,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:17,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:18,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:19,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:20,410 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:20,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:21,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:22,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:23,434 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:24,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:25,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:26,315 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:26,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:27,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:28,051 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:57:28,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:57:28,406 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:57:28,453 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:28,996 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:29,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:30,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:31,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:31,538 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:32,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:33,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:34,436 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:35,439 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:36,051 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:57:36,051 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 with policy now +2025-01-18 19:57:36,056 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:57:36,056 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:57:36,057 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:57:36,057 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:57:36,312 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/3nrtrf6b-media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 +2025-01-18 19:57:36,438 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:36,998 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:36,998 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:57:36,999 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 +2025-01-18 19:57:36,999 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:57:37,069 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:37,284 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/1puhry0r-media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 +2025-01-18 19:57:37,440 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:38,439 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:38,999 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:39,451 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:40,440 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:41,000 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:41,448 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:42,446 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:42,483 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:43,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:57:43,408 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:57:43,446 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:44,457 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:45,001 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:45,451 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:46,448 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:47,447 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:48,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:48,450 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:49,456 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:50,470 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:50,701 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:57:50,701 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:57:50,701 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:57:51,003 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:51,003 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:57:51,004 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:57:51,303 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/p5shhvgr-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:57:51,452 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:52,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:53,004 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:53,421 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:53,464 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:54,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:55,004 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:57:55,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:56,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:57,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:58,051 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:57:58,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:57:58,406 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:57:58,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:57:58,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:57:59,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:00,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:01,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:02,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:03,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:04,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:04,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:05,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:06,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:07,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:08,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:09,412 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:09,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:10,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:11,458 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:12,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:13,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:58:13,406 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:58:13,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:14,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:14,945 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:15,011 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:15,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:16,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:17,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:18,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:19,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:20,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:20,459 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:21,485 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:22,320 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:58:22,320 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 with policy now +2025-01-18 19:58:22,323 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:58:22,323 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:58:22,323 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:58:22,324 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:58:22,461 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:22,571 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/on9jey56-media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 +2025-01-18 19:58:23,014 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:23,014 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:58:23,014 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 +2025-01-18 19:58:23,014 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:58:23,303 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ih02u9zo-media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 +2025-01-18 19:58:23,467 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:24,479 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:25,015 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:25,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:25,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:26,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:27,015 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:27,486 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:28,052 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:58:28,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:58:28,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:58:28,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:29,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:30,502 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:30,881 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:31,017 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:31,498 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:32,017 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:58:32,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:32,751 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:58:32,751 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:58:32,751 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:58:33,017 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:33,018 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:58:33,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:34,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:35,018 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:35,500 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:35,921 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:36,496 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:37,019 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:37,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:38,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:39,020 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:58:39,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:40,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:41,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:41,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:42,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:43,413 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:58:43,414 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:58:43,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:44,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:45,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:46,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:46,517 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:47,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:48,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:49,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:50,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:51,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:52,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:52,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:53,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:54,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:55,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:56,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:57,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:58:57,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:58,052 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:58:58,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:58:58,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:58:58,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:58:59,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:00,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:01,492 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:02,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:02,492 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:03,492 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:04,504 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:04,665 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:59:04,665 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 with policy now +2025-01-18 19:59:04,673 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:59:04,674 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:59:04,674 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:59:04,675 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:59:04,936 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ibqypos3-media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 +2025-01-18 19:59:05,029 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:05,029 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:59:05,029 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 +2025-01-18 19:59:05,029 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:59:05,299 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/b8yu4iz5-media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 +2025-01-18 19:59:05,499 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:06,508 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:07,030 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:07,511 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:07,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:08,508 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:09,504 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:10,507 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:11,031 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:11,503 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:12,508 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:13,032 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:13,289 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:13,438 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:59:13,438 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:59:13,525 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:14,522 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:15,509 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:16,520 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:17,524 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:18,441 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:18,524 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:19,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:20,517 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:21,521 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:22,532 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:22,810 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:59:22,810 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:59:22,810 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:59:23,035 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:59:23,035 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:59:23,290 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/rhopy0zc-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 19:59:23,532 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:23,868 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:24,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:25,036 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:25,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:26,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:27,532 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:28,052 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:59:28,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:59:28,414 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:59:28,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:29,037 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:29,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:29,567 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:30,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:31,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:32,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:33,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:34,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:35,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:35,533 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:36,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:37,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:38,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:39,041 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:39,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:40,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:40,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:41,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:42,534 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:43,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:59:43,414 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:59:43,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:44,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:45,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:45,550 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:46,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:47,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:48,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:49,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:50,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:51,418 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:51,536 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:52,536 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:53,536 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:54,541 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:55,047 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:55,106 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 19:59:55,106 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 with policy now +2025-01-18 19:59:55,118 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:59:55,119 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 19:59:55,119 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:59:55,119 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:59:55,380 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/4idcojoq-media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 +2025-01-18 19:59:55,541 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:56,047 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 19:59:56,047 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 +2025-01-18 19:59:56,048 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 19:59:56,349 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/8k3jrrr9-media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 +2025-01-18 19:59:56,538 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:57,048 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 19:59:57,175 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:59:57,548 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:58,053 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 19:59:58,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:59:58,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:59:58,540 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:59:59,549 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:00,556 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:01,049 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:01,569 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:02,556 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:02,761 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:03,050 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:03,559 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:04,558 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:04,861 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:00:04,861 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:00:04,861 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:00:05,051 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:00:05,051 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:00:05,560 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:06,576 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:07,051 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:07,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:07,925 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:08,567 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:09,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:10,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:11,053 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:11,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:12,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:13,239 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:13,414 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:00:13,414 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:00:13,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:14,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:15,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:16,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:17,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:18,429 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:18,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:19,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:20,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:21,056 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:21,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:22,562 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:23,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:23,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:24,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:25,058 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:25,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:26,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:27,059 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:27,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:28,053 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:00:28,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:00:28,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:00:28,532 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:28,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:29,060 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:29,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:30,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:31,563 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:32,564 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:33,061 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:33,564 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:34,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:34,564 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:35,564 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:36,581 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:37,063 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:37,570 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:37,641 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:00:37,641 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:00:37,642 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 with policy now +2025-01-18 20:00:37,642 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:00:37,642 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:00:37,642 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:00:37,927 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/gpirp6zb-media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 +2025-01-18 20:00:38,063 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:00:38,063 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 +2025-01-18 20:00:38,063 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:00:38,337 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/epmkbwxg-media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 +2025-01-18 20:00:38,581 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:39,064 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:39,439 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:39,573 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:40,583 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:41,586 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:42,591 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:43,065 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:43,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:00:43,430 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:00:43,581 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:44,575 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:45,066 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:45,353 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:45,591 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:46,590 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:47,585 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:48,590 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:48,920 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:00:48,920 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:00:48,921 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:00:49,068 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:49,068 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:00:49,068 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:00:49,321 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/fo54zghb-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:00:49,593 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:50,377 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:50,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:51,069 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:51,596 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:52,597 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:53,069 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:53,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:54,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:55,451 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:00:55,602 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:56,597 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:57,071 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:00:57,597 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:58,054 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:00:58,423 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:00:58,423 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:00:58,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:00:59,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:00,578 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:00,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:01,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:02,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:03,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:04,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:05,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:06,437 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:06,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:07,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:08,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:09,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:10,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:11,437 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:11,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:12,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:13,423 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:01:13,423 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:01:13,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:14,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:15,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:16,558 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:16,599 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:17,600 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:18,600 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:19,602 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:20,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:21,080 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:21,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:22,161 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:22,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:23,628 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:24,206 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:01:24,206 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 with policy now +2025-01-18 20:01:24,210 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:01:24,210 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:01:24,210 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:01:24,210 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:01:24,461 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/845kbyic-media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 +2025-01-18 20:01:24,603 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:25,081 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:25,081 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:01:25,082 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 +2025-01-18 20:01:25,082 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:01:25,403 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/x9ppir5n-media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 +2025-01-18 20:01:25,630 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:26,631 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:27,082 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:27,255 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:27,644 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:28,054 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:01:28,432 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:01:28,432 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:01:28,637 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:29,083 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:29,632 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:30,641 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:31,642 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:32,653 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:32,853 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:33,084 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:33,644 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:34,643 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:34,977 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:01:34,978 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:01:34,978 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:01:35,085 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:01:35,085 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:01:35,656 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:36,658 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:37,086 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:37,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:37,877 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:38,671 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:39,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:40,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:41,088 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:01:41,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:42,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:43,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:43,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:01:43,431 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:01:43,645 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:44,646 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:45,646 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:46,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:47,646 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:48,444 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:48,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:49,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:50,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:51,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:52,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:53,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:53,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:54,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:55,647 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:56,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:57,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:58,055 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:01:58,430 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:01:58,431 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:01:58,585 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:01:58,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:01:59,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:00,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:01,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:02,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:03,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:04,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:04,649 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:05,649 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:06,665 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:07,097 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:07,217 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:02:07,217 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 with policy now +2025-01-18 20:02:07,228 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:02:07,228 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:02:07,229 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:02:07,229 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:02:07,469 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/wttdunw1-media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 +2025-01-18 20:02:07,662 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:08,098 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:02:08,098 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 +2025-01-18 20:02:08,098 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:02:08,370 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/4uqktciz-media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 +2025-01-18 20:02:08,662 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:09,098 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:09,460 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:09,666 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:10,679 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:11,689 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:12,689 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:13,100 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:13,455 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:02:13,455 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:02:13,697 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:14,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:15,033 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:15,101 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:15,697 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:16,722 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:17,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:18,029 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:02:18,029 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:02:18,029 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:02:18,102 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:02:18,102 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:02:18,383 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/rjc30w1s-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:02:18,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:19,102 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:19,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:20,727 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:20,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:21,103 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:21,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:22,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:23,104 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:23,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:24,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:25,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:26,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:26,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:27,698 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:28,055 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:02:28,432 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:02:28,432 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:02:28,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:29,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:30,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:31,503 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:31,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:32,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:33,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:34,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:35,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:36,699 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:37,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:37,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:38,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:39,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:40,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:41,700 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:42,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:42,701 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:43,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:02:43,433 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:02:43,701 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:44,701 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:45,701 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:46,702 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:47,505 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:47,702 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:48,702 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:49,725 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:50,381 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:02:50,381 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 with policy now +2025-01-18 20:02:50,415 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:02:50,415 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:02:50,415 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:02:50,415 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:02:50,671 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ikkwxibk-media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 +2025-01-18 20:02:50,724 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:51,115 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:51,115 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:02:51,115 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 +2025-01-18 20:02:51,115 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:02:51,402 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/jhlb5rn9-media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 +2025-01-18 20:02:51,725 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:52,726 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:53,116 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:53,474 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:53,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:54,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:55,116 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:55,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:56,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:57,735 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:58,056 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:02:58,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:02:58,445 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:02:58,579 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:02:58,754 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:02:59,118 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:02:59,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:00,753 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:01,082 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:03:01,082 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:03:01,083 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:03:01,119 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:03:01,119 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:03:01,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:02,756 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:03,119 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:03,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:04,499 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:04,754 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:05,774 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:06,764 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:07,121 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:07,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:08,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:09,510 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:09,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:10,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:11,123 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:11,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:12,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:13,433 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:03:13,433 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:03:13,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:14,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:14,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:15,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:16,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:17,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:18,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:19,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:20,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:20,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:21,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:22,749 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:23,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:24,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:25,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:25,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:26,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:27,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:28,056 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:03:28,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:03:28,435 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:03:28,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:29,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:30,506 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:30,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:31,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:32,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:33,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:34,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:35,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:36,494 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:36,751 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:37,776 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:38,111 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:03:38,111 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 with policy now +2025-01-18 20:03:38,119 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:03:38,119 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:03:38,119 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:03:38,120 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:03:38,133 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:03:38,134 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 +2025-01-18 20:03:38,134 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:03:38,379 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/silop977-media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 +2025-01-18 20:03:38,642 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/soaimu9s-media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 +2025-01-18 20:03:38,768 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:39,134 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:39,769 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:40,803 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:41,135 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:41,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:41,789 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:42,790 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:43,135 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:43,469 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:03:43,469 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:03:43,797 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:44,798 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:45,816 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:46,845 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:46,937 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:47,137 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:47,821 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:48,803 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:49,796 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:50,140 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:03:50,141 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:03:50,141 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:03:50,142 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:03:50,142 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:03:50,384 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ncdu7a72-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:03:50,855 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:51,142 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:51,842 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:52,449 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:52,813 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:53,822 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:54,845 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:55,144 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:03:55,842 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:56,841 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:57,565 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:03:57,854 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:58,057 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:03:58,470 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:03:58,470 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:03:58,838 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:03:59,840 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:00,829 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:01,146 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:01,829 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:02,829 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:03,097 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:03,829 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:04,830 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:05,830 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:06,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:07,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:08,516 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:08,832 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:09,833 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:10,833 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:11,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:12,833 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:13,438 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:04:13,438 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:04:13,516 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:13,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:14,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:15,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:16,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:17,834 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:18,516 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:18,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:19,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:20,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:21,835 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:22,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:23,517 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:23,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:24,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:25,836 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:26,837 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:27,857 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:28,057 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:04:28,394 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:04:28,394 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 with policy now +2025-01-18 20:04:28,417 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:04:28,417 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:04:28,417 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:04:28,418 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:04:28,470 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:04:28,470 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:04:28,538 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:28,661 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/447rwsgw-media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 +2025-01-18 20:04:28,859 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:29,157 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:29,157 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:04:29,157 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 +2025-01-18 20:04:29,157 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:04:29,452 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/9vv93n32-media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 +2025-01-18 20:04:29,848 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:30,857 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:31,158 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:31,851 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:32,874 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:33,158 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:33,555 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:33,854 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:34,863 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:35,855 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:36,886 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:37,160 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:37,877 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:38,581 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:38,884 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:39,161 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:39,161 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:04:39,200 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:04:39,200 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:04:39,200 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:04:39,932 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:40,161 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:04:40,888 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:41,161 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:41,917 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:42,878 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:43,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:04:43,445 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:04:43,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:44,579 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:44,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:45,163 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:04:45,878 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:46,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:47,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:48,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:49,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:50,519 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:50,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:51,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:52,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:53,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:54,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:55,519 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:04:55,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:56,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:57,879 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:58,058 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:04:58,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:04:58,445 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:04:58,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:04:59,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:00,582 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:00,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:01,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:02,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:03,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:04,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:05,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:06,520 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:06,880 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:07,881 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:08,881 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:09,932 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:10,907 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:11,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:11,296 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:05:11,296 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 with policy now +2025-01-18 20:05:11,301 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:05:11,301 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:05:11,302 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:05:11,302 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:05:11,554 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/os7081uj-media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 +2025-01-18 20:05:11,918 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:12,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:05:12,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 +2025-01-18 20:05:12,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:05:12,385 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:12,458 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/86r0qhw1-media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 +2025-01-18 20:05:12,929 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:13,174 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:13,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:05:13,482 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:05:13,909 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:14,915 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:15,966 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:16,922 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:17,176 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:17,634 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:17,928 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:18,938 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:19,177 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:19,950 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:20,956 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:21,943 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:22,968 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:23,536 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:23,964 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:24,973 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:25,971 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:26,971 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:27,976 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:28,058 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:05:28,495 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:05:28,496 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:05:28,650 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:28,996 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:29,964 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:30,982 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:31,969 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:32,981 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:34,000 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:34,551 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:34,998 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:35,971 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:36,994 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:38,007 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:38,975 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:39,598 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:39,997 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:40,991 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:41,992 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:43,029 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:43,471 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:05:43,472 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:05:43,999 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:44,801 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:45,006 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:46,040 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:47,019 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:48,018 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:49,033 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:50,031 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:50,572 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:51,044 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:52,048 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:53,050 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:54,078 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:55,083 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:55,578 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:05:56,070 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:57,085 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:57,192 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:05:57,192 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:05:57,298 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:05:57,298 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:05:57,298 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:05:58,058 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:05:58,081 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:58,192 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:05:58,480 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:05:58,480 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:05:59,085 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:05:59,193 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:00,090 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:00,849 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:01,083 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:02,140 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:03,092 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:03,194 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:04,113 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:05,112 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:06,114 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:06,579 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:07,112 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:08,130 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:09,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:10,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:11,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:11,197 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:12,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:12,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:13,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:13,474 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:06:13,474 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:06:14,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:15,105 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:16,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:17,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:17,579 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:18,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:19,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:20,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:21,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:22,106 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:23,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:23,572 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:24,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:25,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:26,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:27,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:28,059 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:06:28,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:28,474 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:06:28,474 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:06:29,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:29,572 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:30,107 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:31,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:32,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:33,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:34,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:34,572 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:35,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:36,108 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:37,133 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:37,209 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:37,521 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:06:37,521 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 with policy now +2025-01-18 20:06:37,545 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:06:37,546 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:06:37,546 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:06:37,546 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:06:37,782 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/gm0yq7b0-media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 +2025-01-18 20:06:38,124 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:38,210 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:06:38,210 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 +2025-01-18 20:06:38,210 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:06:38,499 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/26qywfdv-media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 +2025-01-18 20:06:39,137 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:39,210 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:39,588 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:40,150 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:41,136 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:42,153 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:43,146 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:43,212 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:43,492 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:06:43,492 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:06:44,164 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:45,153 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:45,212 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:45,497 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:46,171 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:47,151 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:48,176 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:49,146 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:49,361 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:06:49,361 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:06:49,366 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:06:50,170 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:50,214 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:06:50,215 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:06:50,469 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/x6a11xcl-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:06:50,550 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:51,174 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:51,215 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:52,183 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:53,188 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:54,172 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:55,175 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:55,217 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:06:55,771 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:06:56,171 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:57,172 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:58,059 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:06:58,174 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:06:58,496 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:06:58,497 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:06:59,190 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:00,221 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:01,202 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:01,617 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:02,201 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:03,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:03,220 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:04,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:05,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:06,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:07,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:07,594 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:08,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:09,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:10,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:11,197 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:12,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:13,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:13,224 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:13,373 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:13,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:07:13,483 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:07:14,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:15,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:16,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:17,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:18,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:18,594 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:19,198 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:20,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:21,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:22,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:23,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:24,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:24,594 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:25,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:26,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:27,199 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:28,060 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:07:28,200 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:28,482 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:07:28,483 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:07:29,203 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:30,230 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:30,305 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:30,978 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:07:30,978 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 with policy now +2025-01-18 20:07:30,998 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:07:30,999 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:07:30,999 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:07:30,999 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:07:31,215 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:31,232 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:31,232 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:07:31,232 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 +2025-01-18 20:07:31,233 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:07:31,250 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/ojwd1tbi-media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 +2025-01-18 20:07:31,517 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/fp2c2bo7-media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 +2025-01-18 20:07:32,235 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:33,233 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:33,234 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:34,233 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:34,275 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:35,262 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:35,713 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:36,234 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:36,248 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:37,271 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:38,233 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:39,265 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:40,237 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:40,237 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:07:40,237 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:40,426 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:07:40,426 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:07:40,426 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:07:41,238 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:07:41,238 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:07:41,259 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:41,353 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:42,239 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:42,244 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:43,261 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:43,504 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:07:43,504 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:07:44,239 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:44,294 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:45,293 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:46,240 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:46,314 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:46,686 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:47,301 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:48,292 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:49,298 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:50,303 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:51,293 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:52,306 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:52,643 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:53,291 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:54,336 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:55,281 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:56,244 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:07:56,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:57,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:58,060 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:07:58,061 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:07:58,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:07:58,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:07:58,488 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:07:59,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:00,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:01,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:02,247 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:02,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:03,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:03,445 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:04,282 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:05,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:06,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:07,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:08,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:08,616 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:09,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:10,250 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:10,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:11,283 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:12,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:13,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:13,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:08:13,488 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:08:14,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:14,616 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:15,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:16,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:17,284 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:18,285 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:19,285 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:19,616 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:20,285 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:21,285 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:22,290 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:22,534 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:08:22,535 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 with policy now +2025-01-18 20:08:22,554 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:08:22,554 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:08:22,554 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:08:22,555 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:08:22,807 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/9oi6gffe-media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 +2025-01-18 20:08:23,256 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:08:23,256 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 +2025-01-18 20:08:23,256 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:08:23,343 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:23,596 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/zjpz506b-media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 +2025-01-18 20:08:24,256 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:24,301 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:24,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:25,257 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:25,339 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:26,325 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:27,329 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:28,061 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:08:28,258 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:28,316 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:28,556 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:08:28,556 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:08:29,355 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:29,669 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:30,354 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:31,357 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:32,260 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:32,383 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:32,487 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:08:32,488 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:08:32,488 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:08:33,260 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:08:33,260 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:08:33,362 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:34,261 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:34,348 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:34,689 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:35,390 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:36,261 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:36,357 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:37,393 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:38,262 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:38,377 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:39,377 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:40,263 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:40,387 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:40,644 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:41,389 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:42,392 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:43,419 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:43,540 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:08:43,540 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:08:44,404 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:45,428 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:45,678 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:46,431 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:47,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:48,267 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:48,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:49,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:50,267 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:08:50,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:51,405 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:51,623 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:52,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:53,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:54,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:55,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:56,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:56,623 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:08:57,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:58,061 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:08:58,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:08:58,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:08:58,501 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:08:59,406 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:00,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:01,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:02,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:02,623 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:03,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:04,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:05,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:06,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:07,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:07,623 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:08,407 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:09,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:10,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:11,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:12,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:13,086 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:13,408 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:13,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:09:13,501 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:09:14,278 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:14,435 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:15,420 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:15,798 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:09:15,798 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 with policy now +2025-01-18 20:09:15,802 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:09:15,802 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:09:15,802 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:09:15,802 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:09:16,073 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/dxyaze9m-media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 +2025-01-18 20:09:16,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:16,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:09:16,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 +2025-01-18 20:09:16,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:09:16,453 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:16,551 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/s8tnjemt-media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 +2025-01-18 20:09:17,279 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:17,454 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:18,280 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:18,478 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:18,663 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:19,443 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:20,452 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:21,477 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:22,282 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:22,462 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:23,465 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:23,665 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:24,282 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:24,450 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:25,475 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:26,471 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:27,284 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:09:27,452 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:27,541 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/53vvgyah-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:09:27,555 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:09:27,555 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:09:27,555 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:09:28,061 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:09:28,284 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:28,284 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:09:28,483 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:28,545 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:09:28,545 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:09:28,675 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:29,470 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:30,285 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:30,543 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:31,541 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:32,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:33,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:34,287 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:34,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:34,663 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:35,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:36,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:37,487 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:38,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:39,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:39,664 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:40,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:41,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:42,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:43,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:43,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:09:43,501 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:09:44,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:45,488 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:45,664 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:46,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:47,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:48,293 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:48,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:49,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:50,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:50,664 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:51,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:52,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:53,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:54,489 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:55,490 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:55,664 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:09:56,491 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:57,493 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:58,062 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:09:58,298 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:09:58,493 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:09:58,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:09:58,502 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:09:59,501 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:00,299 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:00,299 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 +2025-01-18 20:10:00,299 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:10:00,338 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:10:00,338 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 with policy now +2025-01-18 20:10:00,362 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:10:00,362 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:10:00,362 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:10:00,363 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:10:00,502 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:00,565 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/4vutdphb-media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 +2025-01-18 20:10:00,849 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/nmxvdykm-media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 +2025-01-18 20:10:01,299 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:10:01,476 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:01,524 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:02,300 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:02,539 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:03,300 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:03,561 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:04,586 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:05,551 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:06,302 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:06,551 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:07,181 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:07,578 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:08,602 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:09,601 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:09,606 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:10:09,606 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:10:09,607 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:10:10,303 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:10,304 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:10:10,304 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:10:10,611 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:11,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:12,304 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:12,629 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:12,661 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:13,535 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:10:13,536 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:10:13,640 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:14,607 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:15,607 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:16,306 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:16,607 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:17,607 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:18,209 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:18,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:19,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:20,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:21,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:22,309 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:22,608 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:23,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:23,680 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:24,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:25,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:26,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:27,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:28,062 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:10:28,506 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:10:28,507 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:10:28,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:28,680 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:29,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:30,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:31,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:32,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:33,609 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:33,680 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:34,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:35,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:36,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:37,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:38,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:38,680 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:39,610 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:40,611 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:41,654 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:42,318 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:42,613 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:10:42,613 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 with policy now +2025-01-18 20:10:42,622 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:42,669 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:10:42,669 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:10:42,670 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:10:42,670 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:10:42,863 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/w2hj5yjy-media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 +2025-01-18 20:10:43,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:43,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:10:43,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 +2025-01-18 20:10:43,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:10:43,541 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:10:43,541 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:10:43,604 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/hh67nec3-media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 +2025-01-18 20:10:43,622 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:43,722 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:44,319 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:44,659 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:45,320 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:45,662 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:46,648 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:47,642 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:48,321 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:48,653 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:48,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:49,674 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:50,703 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:51,688 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:52,323 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:52,685 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:53,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:54,720 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:10:54,750 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:55,735 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:56,664 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:10:56,664 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:10:56,664 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:10:56,714 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:57,325 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:10:57,325 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:10:57,748 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:58,063 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:10:58,325 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:10:58,542 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:10:58,542 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:10:58,715 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:10:59,719 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:00,326 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:00,719 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:00,719 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:01,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:02,328 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:02,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:03,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:04,329 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:04,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:05,706 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:06,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:06,710 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:07,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:08,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:09,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:10,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:11,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:11,710 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:12,707 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:13,508 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:11:13,509 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:11:13,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:14,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:15,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:16,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:16,710 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:17,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:18,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:19,708 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:20,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:21,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:21,711 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:22,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:23,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:24,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:25,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:26,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:26,711 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:27,709 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:28,063 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:11:28,546 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:11:28,546 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:11:28,729 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:29,340 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:11:29,340 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 with policy now +2025-01-18 20:11:29,343 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 +2025-01-18 20:11:29,343 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval +2025-01-18 20:11:29,348 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:11:29,348 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:11:29,348 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:29,349 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:29,598 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/q9wwsyvm-media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 +2025-01-18 20:11:29,716 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:29,856 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/8dqa5vaf-media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 +2025-01-18 20:11:30,343 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:30,343 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:11:30,722 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:31,718 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:32,344 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:32,411 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:32,774 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:33,737 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:34,346 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:34,723 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:35,762 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:36,772 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:37,523 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:37,763 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:38,347 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:38,764 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:39,348 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:11:39,720 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:11:39,720 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:39,721 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:39,804 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:40,348 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:40,349 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:11:40,522 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 20:11:40,522 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 20:11:40,717 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:11:40,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:41,012 INFO wandb-upload_3:2566524 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu0o677tu +2025-01-18 20:11:41,074 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 20:11:41,085 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:11:41,349 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:11:41,349 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json +2025-01-18 20:11:41,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json +2025-01-18 20:11:41,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media +2025-01-18 20:11:41,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval +2025-01-18 20:11:41,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table +2025-01-18 20:11:41,402 INFO SenderThread:2566524 [sender.py:send_request_log_artifact():1525] logged artifact run-mo0b6mh9-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTk1Mzc3Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA2Mzk2Nw==', 'latestArtifact': None}} +2025-01-18 20:11:41,402 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:11:41,403 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json with policy now +2025-01-18 20:11:41,403 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:11:41,403 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,403 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,404 DEBUG SenderThread:2566524 [sender.py:send():379] send: telemetry +2025-01-18 20:11:41,404 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,407 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,408 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,408 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,408 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,409 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,409 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,409 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,409 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,410 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,410 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,410 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,410 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,411 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,411 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,411 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,412 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,412 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,412 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,412 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,413 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:41,413 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:41,413 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 20:11:41,623 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/yzdnup9g-media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json +2025-01-18 20:11:41,644 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/24g65024-media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json +2025-01-18 20:11:41,661 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/sr481jlx-events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:11:41,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:41,782 INFO wandb-upload_3:2566524 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe6tbkd_4 +2025-01-18 20:11:41,871 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/xi840sew-media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json +2025-01-18 20:11:42,179 INFO SenderThread:2566524 [sender.py:send_request_log_artifact():1525] logged artifact run-mo0b6mh9-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTk1Mzc4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA2Mzk2OA==', 'latestArtifact': None}} +2025-01-18 20:11:42,179 DEBUG SenderThread:2566524 [sender.py:send():379] send: files +2025-01-18 20:11:42,179 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json with policy now +2025-01-18 20:11:42,180 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:11:42,180 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:11:42,180 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:11:42,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:11:42,428 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /tmp/tmpwgaaf6o2wandb/44lpp5dk-media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json +2025-01-18 20:11:42,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:43,185 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:43,350 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml +2025-01-18 20:11:43,513 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:11:43,513 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:11:43,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:44,351 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:11:44,738 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:45,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:46,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:47,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:48,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:48,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:49,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:50,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:51,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:52,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:53,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:53,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:54,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:55,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:56,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:57,740 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:58,064 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:11:58,513 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:11:58,513 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:11:58,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:11:58,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:11:59,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:00,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:01,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:02,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:03,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:04,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:04,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:05,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:06,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:07,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:08,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:09,739 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:09,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:10,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:11,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:12,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:13,513 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:12:13,513 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:12:13,743 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:14,021 DEBUG SenderThread:2566524 [sender.py:send():379] send: exit +2025-01-18 20:12:14,021 INFO SenderThread:2566524 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 20:12:14,021 INFO SenderThread:2566524 [sender.py:send_exit():588] handling runtime: 1035 +2025-01-18 20:12:14,022 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:12:14,022 INFO SenderThread:2566524 [sender.py:send_exit():594] send defer +2025-01-18 20:12:14,022 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,022 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 20:12:14,022 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:14,022 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 20:12:14,022 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 1 +2025-01-18 20:12:14,022 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,023 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 20:12:14,023 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:14,023 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 20:12:14,023 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 2 +2025-01-18 20:12:14,023 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,023 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 20:12:14,023 INFO HandlerThread:2566524 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 20:12:14,023 DEBUG SystemMonitor:2566524 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 20:12:14,023 DEBUG SystemMonitor:2566524 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 20:12:14,023 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 20:12:14,024 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 20:12:14,074 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 20:12:14,074 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 20:12:14,074 INFO HandlerThread:2566524 [interfaces.py:finish():200] Joined network monitor +2025-01-18 20:12:14,074 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:14,074 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 20:12:14,074 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 3 +2025-01-18 20:12:14,075 DEBUG SenderThread:2566524 [sender.py:send():379] send: stats +2025-01-18 20:12:14,075 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,075 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 20:12:14,075 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:14,075 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 20:12:14,075 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 4 +2025-01-18 20:12:14,075 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:14,075 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 20:12:14,366 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:12:16,367 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:12:20,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:20,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:20,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 20:12:20,730 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:20,731 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:20,731 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 20:12:20,731 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 5 +2025-01-18 20:12:20,731 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:20,732 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:20,732 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 20:12:20,740 DEBUG SenderThread:2566524 [sender.py:send():379] send: history +2025-01-18 20:12:20,740 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:12:20,740 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:12:20,741 DEBUG SenderThread:2566524 [sender.py:send():379] send: summary +2025-01-18 20:12:20,741 INFO SenderThread:2566524 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:12:20,741 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:20,741 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 20:12:20,741 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 6 +2025-01-18 20:12:20,741 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:20,741 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 20:12:20,742 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:20,742 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 20:12:20,742 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 7 +2025-01-18 20:12:20,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:12:20,742 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:20,742 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 20:12:20,742 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:20,742 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 20:12:21,022 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:21,370 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:12:22,841 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 8 +2025-01-18 20:12:22,841 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:22,842 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:22,842 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 20:12:22,842 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:22,842 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 20:12:22,842 INFO SenderThread:2566524 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 20:12:22,845 INFO SenderThread:2566524 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 20:12:22,846 INFO SenderThread:2566524 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 20:12:22,861 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 9 +2025-01-18 20:12:22,861 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:22,861 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 20:12:22,862 DEBUG SenderThread:2566524 [sender.py:send():379] send: artifact +2025-01-18 20:12:23,023 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:23,239 INFO wandb-upload_2:2566524 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe9xdfu6j +2025-01-18 20:12:23,239 INFO wandb-upload_0:2566524 [upload_job.py:push():85] Skipped uploading /tmp/tmp6zpygqk6/wandb-job.json +2025-01-18 20:12:23,359 INFO wandb-upload_3:2566524 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpi1brqpjv +2025-01-18 20:12:23,371 INFO Thread-12 :2566524 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:12:23,938 INFO SenderThread:2566524 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTk1NDMyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTkxOTkxNA==', 'versionIndex': 448}}} +2025-01-18 20:12:23,938 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:23,938 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 20:12:23,938 INFO SenderThread:2566524 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 20:12:24,373 INFO SenderThread:2566524 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files +2025-01-18 20:12:24,373 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml config.yaml +2025-01-18 20:12:24,373 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 events.out.tfevents.1737248098.leonmkim-ROG-Strix-G15CS-G15CS.2566358.0 +2025-01-18 20:12:24,373 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json wandb-summary.json +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/requirements.txt requirements.txt +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-metadata.json wandb-metadata.json +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log output.log +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/diff.patch diff.patch +2025-01-18 20:12:24,375 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 media/videos/eval/5_eval_5_d80af6b216e8c15164e9.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 media/videos/eval/2_eval_2_593caa1fa9ea606eafa5.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 media/videos/eval/14_eval_14_bf5561b5a4f98495a054.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 media/videos/eval/6_eval_6_3602d914de1709b10240.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 media/videos/eval/16_eval_16_7e356bbb63bbb79e9934.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 media/videos/eval/17_eval_17_21895a4eb18f20222f93.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 media/videos/eval/0_eval_0_dd640c135955b3d54fec.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 media/videos/eval/10_eval_10_708e5d86baaf5c876d21.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 media/videos/eval/8_eval_8_4a18bdd252b8f5e209e5.mp4 +2025-01-18 20:12:24,380 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 media/videos/eval/4_eval_4_09e9e69382c41ac17586.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 media/videos/eval/13_eval_13_ce205101458cc00950f1.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 media/videos/eval/12_eval_12_7a56ddf9bcd287e59382.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 media/videos/eval/15_eval_15_2535ee93601a7918b8bf.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 media/videos/eval/11_eval_11_5e48fceb469367c5942f.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 media/videos/eval/1_eval_1_f01db884f36bf3d225fe.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 media/videos/eval/7_eval_7_960ad0adbe0863fa34f9.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 media/videos/eval/19_eval_19_0dc41dd83150fd316589.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 media/videos/eval/3_eval_3_05651e72bfc1fe624e25.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 media/videos/eval/9_eval_9_eeb6222427bdaa35d11b.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 media/videos/eval/18_eval_18_76a7bdc1f11678727d6f.mp4 +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json media/table/eval/success_rate_ci_20_6cabaaf3e79ec48751e3.table.json +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json media/table/eval/total_success_rate_ci_21_109ab103ab9a05e5d575.table.json +2025-01-18 20:12:24,381 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 10 +2025-01-18 20:12:24,382 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:24,382 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:24,382 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 20:12:24,382 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:24,382 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 20:12:24,382 INFO SenderThread:2566524 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 20:12:24,566 INFO wandb-upload_1:2566524 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/config.yaml +2025-01-18 20:12:24,581 INFO wandb-upload_3:2566524 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/output.log +2025-01-18 20:12:24,583 INFO wandb-upload_2:2566524 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/requirements.txt +2025-01-18 20:12:24,589 INFO wandb-upload_0:2566524 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/files/wandb-summary.json +2025-01-18 20:12:24,789 INFO Thread-11 (_thread_body):2566524 [sender.py:transition_state():614] send defer: 11 +2025-01-18 20:12:24,789 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:24,789 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 20:12:24,790 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:24,790 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 20:12:24,790 INFO SenderThread:2566524 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 20:12:24,793 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 12 +2025-01-18 20:12:24,793 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:24,793 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 20:12:24,793 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:24,793 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 20:12:24,793 INFO SenderThread:2566524 [file_stream.py:finish():601] file stream finish called +2025-01-18 20:12:25,014 INFO SenderThread:2566524 [file_stream.py:finish():605] file stream finish is done +2025-01-18 20:12:25,014 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 13 +2025-01-18 20:12:25,014 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:25,014 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 20:12:25,015 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:25,015 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 20:12:25,015 INFO SenderThread:2566524 [sender.py:transition_state():614] send defer: 14 +2025-01-18 20:12:25,015 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:12:25,015 INFO HandlerThread:2566524 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 20:12:25,015 DEBUG SenderThread:2566524 [sender.py:send():379] send: final +2025-01-18 20:12:25,015 DEBUG SenderThread:2566524 [sender.py:send():379] send: footer +2025-01-18 20:12:25,015 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: defer +2025-01-18 20:12:25,015 INFO SenderThread:2566524 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 20:12:25,016 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:25,016 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 20:12:25,016 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:25,016 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:12:25,016 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: server_info +2025-01-18 20:12:25,017 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 20:12:25,018 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 20:12:25,019 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:12:25,078 DEBUG SenderThread:2566524 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:12:25,078 INFO MainThread:2566524 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 20:12:25,079 INFO MainThread:2566524 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 20:12:25,079 INFO MainThread:2566524 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 20:12:25,080 DEBUG HandlerThread:2566524 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 20:12:25,080 INFO HandlerThread:2566524 [handler.py:finish():882] shutting down handler +2025-01-18 20:12:26,017 INFO WriterThread:2566524 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/run-mo0b6mh9.wandb +2025-01-18 20:12:26,078 INFO SenderThread:2566524 [sender.py:finish():1615] shutting down sender +2025-01-18 20:12:26,079 INFO SenderThread:2566524 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 20:12:26,079 INFO SenderThread:2566524 [file_pusher.py:join():175] waiting for file pusher diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug.log b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8969ed10d7699ba2a7f1de0c4eb48e72a38a10ef --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Configure stats pid to 2566358 +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/settings +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug.log +2025-01-18 19:54:57,681 INFO MainThread:2566358 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/logs/debug-internal.log +2025-01-18 19:54:57,682 INFO MainThread:2566358 [wandb_init.py:init():569] calling init triggers +2025-01-18 19:54:57,682 INFO MainThread:2566358 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9'} +2025-01-18 19:54:57,682 INFO MainThread:2566358 [wandb_init.py:init():619] starting backend +2025-01-18 19:54:57,682 INFO MainThread:2566358 [wandb_init.py:init():623] setting up manager +2025-01-18 19:54:57,687 INFO MainThread:2566358 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 19:54:57,688 INFO MainThread:2566358 [wandb_init.py:init():631] backend started and connected +2025-01-18 19:54:57,700 INFO MainThread:2566358 [wandb_init.py:init():720] updated telemetry +2025-01-18 19:54:57,707 INFO MainThread:2566358 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 19:54:57,962 INFO MainThread:2566358 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 19:54:58,032 INFO MainThread:2566358 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 19:54:58,032 INFO MainThread:2566358 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 19:54:58,403 INFO MainThread:2566358 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 19:54:58,403 INFO MainThread:2566358 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 19:54:58,403 INFO MainThread:2566358 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 19:54:58,404 INFO MainThread:2566358 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 19:54:58,405 INFO MainThread:2566358 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 19:54:58,405 INFO MainThread:2566358 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/mo0b6mh9/tb, True +2025-01-18 19:55:02,836 INFO MainThread:2566358 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 20:12:26,080 WARNING MsgRouterThr:2566358 [router.py:message_loop():77] message_loop has been closed diff --git a/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/run-mo0b6mh9.wandb b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/run-mo0b6mh9.wandb new file mode 100644 index 0000000000000000000000000000000000000000..876e02d5fa843dcfaf85b3c2bf83f0bc82f33805 --- /dev/null +++ b/mo0b6mh9/wandb/run-20250118_195457-mo0b6mh9/run-mo0b6mh9.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f983acd554e43ff7ccbe4cd8ba068f4056bfc7adb262b014bf51b1c1de8ae76f +size 237054 diff --git a/q1ti958r/episode_rosbags/cam_tf_world.npy b/q1ti958r/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/q1ti958r/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/q1ti958r/episode_rosbags/depth_K.npy b/q1ti958r/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/q1ti958r/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/q1ti958r/episode_rosbags/episode_0_2025-01-18-18-50-38.bag b/q1ti958r/episode_rosbags/episode_0_2025-01-18-18-50-38.bag new file mode 100644 index 0000000000000000000000000000000000000000..c5db4bf4e7113d312d88c3aba95c77e2c9328b15 --- /dev/null +++ b/q1ti958r/episode_rosbags/episode_0_2025-01-18-18-50-38.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c96b6b906cbc1dc0e21c187c1814337420be6a5b8ebac6845f2cf1ff41b2b373 +size 1410022682 diff --git a/q1ti958r/episode_rosbags/episode_10_2025-01-18-18-57-52.bag b/q1ti958r/episode_rosbags/episode_10_2025-01-18-18-57-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..d123b37654e558b10bd812eef204557cb53e770e --- /dev/null +++ b/q1ti958r/episode_rosbags/episode_10_2025-01-18-18-57-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4235da9c5941357db79745fc364dd5adab5db24b3a7d665658d263e56a3ad41 +size 1412593636 diff --git a/q1ti958r/episode_rosbags/episode_11_2025-01-18-18-58-35.bag b/q1ti958r/episode_rosbags/episode_11_2025-01-18-18-58-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..ff1032ad68cc3d8a8eccf9edb31237e225b083d2 --- /dev/null +++ b/q1ti958r/episode_rosbags/episode_11_2025-01-18-18-58-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:575080cee8c7001864951bbf1a7c099e4f82c0c38194c6fb5e1fb168eb41b873 +size 1411241610 diff --git a/q1ti958r/episode_rosbags/episode_19_2025-01-18-19-05-54.bag b/q1ti958r/episode_rosbags/episode_19_2025-01-18-19-05-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..21a86d439d5c46b43807ab4213d72422133df0ca --- /dev/null +++ b/q1ti958r/episode_rosbags/episode_19_2025-01-18-19-05-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:228c0113328161e6a200fa92ec83628f82666f256edfcf9d2b86a2c984540c71 +size 1406362147 diff --git a/q1ti958r/eval_robot.log b/q1ti958r/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..3bd35e8be0213058345b8e6d68bdf997a0e8ae94 --- /dev/null +++ b/q1ti958r/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-18 18:47:38,754][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 18:47:38,757][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( +